Recursively re-spawn file on fsnotify Remove/Rename (Golang)

The Goal:

I am attempting to monitor a file that is subject to being moved or deleted at any time. If and when it is, I’d like to re-generate this file so that an app can continue to write to it.

Attempted:

I have attempted to do this by implementing two functions, monitorFile() to listen for fsnotify events and send the removed filename over a channel to listen() which upon receiving the filepath string over un-buffered channel mvrm (move or rename), will recursively re-generate the file.

Observed behavior:

I can echo 'foo' >> ./inlogs/test.log and see a write notification, and can even rm ./inlogs/test.log (or mv) and see that the file is re-generated… but only once. If I rm or mv the file a second time, the file is not re-generated.

• Strangely, the undesired behavior does not occur on local Mac OSx (System Version: macOS 10.13.2 (17C88), Kernel Version: Darwin 17.3.0), but does on two different Linux machines with builds:

Linux 3.13.0-32-generic #57-Ubuntu SMP x86_64 x86_64 x86_64 GNU/Linux

Linux 4.9.51-10.52.amzn1.x86_64 #1 SMP x86_64 x86_64 x86_64 GNU/Linux

Diagnostics Attempted:

The differing behavior makes me think I have a race condition. However go build -race provides no output.

I wonder if done chan is receiving due to such a race condition?

Apologies that this is not ‘Playground-able’, but any advice or observation of where this might by racy or buggy would be welcome.

watcher.go:

package main

import (
    "os"
    "log"
    "fmt"

    "github.com/go-fsnotify/fsnotify"
)

//Globals
var mvrm chan string

func main() {
    mvrm = make(chan string)
    listen(mvrm)
    monitorFile("./inlogs/test.log", mvrm)
}

func listen(mvrm chan string) {
    go func() {
        for {
            select {
            case fileName := <-mvrm :
                fmt.Println(fileName)
                newFile, err := os.OpenFile(fileName, os.O_RDWR | os.O_CREATE | os.O_APPEND , 0666)
                if err == nil {
                    defer newFile.Close()

                    // Recursively re-spawn monitoring
                    go listen(mvrm)
                    go monitorFile(fileName, mvrm)
                } else {
                    log.Fatal("Err re-spawning file")
                }
            default:
                continue
            }
        }
    }()
}

func monitorFile(filepath string, mvrm chan string) {
    watcher, err := fsnotify.NewWatcher()
    if err != nil {
        log.Fatal(err)
    }
    defer watcher.Close()

    done := make(chan bool)
    go func() {
        for {
            select {
            case event := <-watcher.Events:
                switch event.Op {
                case fsnotify.Write :
                    log.Println("Write!")
                    continue
                case fsnotify.Chmod :
                    log.Println("Chmod!")
                    continue
                case fsnotify.Remove, fsnotify.Rename :
                    log.Println("Moved or Deleted!")
                    mvrm <- event.Name
                    continue
                default:
                    log.Printf("Unknown: %vn", event.Op)
                    continue
                }
            case err := <-watcher.Errors:
                log.Println("Error:", err)
            }
        }
    }()

    err = watcher.Add(filepath)
    if err != nil {
        log.Fatal(err)
    }
    <-done
}

EDIT:

With some great feedback, I’ve paired this down. In Linux it is now re-generating the file as intended, but after monitoring with top, I see it is spawning a new PID every time the file is moved or deleted, so I do still have a leak. Suggestions as to how I might eliminate this behavior welcome.

https://play.golang.org/p/FrlkktoK2-s

Answer

Please see the code comments, most of the discussion in code comments.

https://play.golang.com/p/qxq58h1nQjp

Outside the golang universe, but facebook has a tool that does pretty much what you are looking for, just not as much go code fun :): https://github.com/facebook/watchman

package main

import (
    "log"
    "os"

    // couldn't find the go-fsnotify, this is what pops up on github
    "github.com/fsnotify/fsnotify"
)

func main() {
    monitorFile("./inlogs/test.log")
}

func monitorFile(filepath string) {

    // starting watcher
    watcher, err := fsnotify.NewWatcher()
    if err != nil {
        log.Fatal(err)
    }
    defer watcher.Close()

    // monitor events
    go func() {
        for {
            select {
            case event := <-watcher.Events:
                switch event.Op {
                case fsnotify.Create:
                    log.Println("Created")

                case fsnotify.Write:
                    log.Println("Write")

                case fsnotify.Chmod:
                    log.Println("Chmod")

                case fsnotify.Remove, fsnotify.Rename:
                    log.Println("Moved or Deleted")

                    respawnFile(event.Name)

                    // add the file back to watcher, since it is removed from it
                    // when file is moved or deleted
                    log.Printf("add to watcher file:  %sn", filepath)
                    // add appears to be concurrently safe so calling from multiple go routines is ok
                    err = watcher.Add(filepath)
                    if err != nil {
                        log.Fatal(err)
                    }

                    // there is  not need to break the loop
                    // we just continue waiting for events from the same watcher

                }
            case err := <-watcher.Errors:
                log.Println("Error:", err)
            }
        }
    }()

    // add file to the watcher first time
    log.Printf("add to watcher 1st: %sn", filepath)
    err = watcher.Add(filepath)
    if err != nil {
        log.Fatal(err)
    }

    // to keep waiting forever, to prevent main exit
    // this is to replace the done channel
    select {}
}

func respawnFile(filepath string) {
    log.Printf("re creating file %sn", filepath)

    // you just need the os.Create()
    respawned, err := os.Create(filepath)
    if err != nil {
        log.Fatalf("Err re-spawning file: %v", filepath)
    }
    defer respawned.Close()

    // there is no need to call monitorFile again, it never returns
    // the call to "go monitorFile(filepath)" was causing another go routine leak
}

Have fun!