diff --git a/internal/net/http/accesslog/access_logger.go b/internal/net/http/accesslog/access_logger.go
index 7738287..6d493f8 100644
--- a/internal/net/http/accesslog/access_logger.go
+++ b/internal/net/http/accesslog/access_logger.go
@@ -129,7 +129,6 @@ func (l *AccessLogger) Flush(force bool) {
 		l.write(l.buf.Bytes())
 		l.buf.Reset()
 		l.bufMu.Unlock()
-		logger.Debug().Msg("access log flushed to " + l.io.Name())
 	}
 }
 
@@ -170,5 +169,7 @@ func (l *AccessLogger) write(data []byte) {
 	l.io.Unlock()
 	if err != nil {
 		l.handleErr(err)
+	} else {
+		logger.Debug().Msg("access log flushed to " + l.io.Name())
 	}
 }
diff --git a/internal/net/http/accesslog/file_logger.go b/internal/net/http/accesslog/file_logger.go
index 7b3aec7..e2dc993 100644
--- a/internal/net/http/accesslog/file_logger.go
+++ b/internal/net/http/accesslog/file_logger.go
@@ -3,36 +3,69 @@ package accesslog
 import (
 	"fmt"
 	"os"
+	"path"
 	"sync"
 
 	"github.com/yusing/go-proxy/internal/task"
+	"github.com/yusing/go-proxy/internal/utils"
 )
 
 type File struct {
 	*os.File
 	sync.Mutex
+
+	// os.File.Name() may not equal to key of `openedFiles`.
+	// Store it for later delete from `openedFiles`.
+	path string
+
+	refCount *utils.RefCount
 }
 
 var (
-	openedFiles   = make(map[string]AccessLogIO)
+	openedFiles   = make(map[string]*File)
 	openedFilesMu sync.Mutex
 )
 
 func NewFileAccessLogger(parent task.Parent, cfg *Config) (*AccessLogger, error) {
 	openedFilesMu.Lock()
 
-	var io AccessLogIO
-	if opened, ok := openedFiles[cfg.Path]; ok {
-		io = opened
+	var file *File
+	path := path.Clean(cfg.Path)
+	if opened, ok := openedFiles[path]; ok {
+		opened.refCount.Add()
+		file = opened
 	} else {
-		f, err := os.OpenFile(cfg.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+		f, err := os.OpenFile(cfg.Path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o644)
 		if err != nil {
+			openedFilesMu.Unlock()
 			return nil, fmt.Errorf("access log open error: %w", err)
 		}
-		io = &File{File: f}
-		openedFiles[cfg.Path] = io
+		file = &File{File: f, path: path, refCount: utils.NewRefCounter()}
+		openedFiles[path] = file
+		go file.closeOnZero()
 	}
 
 	openedFilesMu.Unlock()
-	return NewAccessLogger(parent, io, cfg), nil
+	return NewAccessLogger(parent, file, cfg), nil
+}
+
+func (f *File) Close() error {
+	f.refCount.Sub()
+	return nil
+}
+
+func (f *File) closeOnZero() {
+	defer logger.Debug().
+		Str("path", f.path).
+		Msg("access log closed")
+	for {
+		select {
+		case <-f.refCount.Zero():
+			openedFilesMu.Lock()
+			delete(openedFiles, f.path)
+			openedFilesMu.Unlock()
+			f.File.Close()
+			return
+		}
+	}
 }