// rss2hook is a simple utility which will make HTTP POST // requests to remote web-hooks when new items appear in an RSS feed. // // Steve // poesty // package main import ( "bytes" "encoding/json" "flag" "io" "log" "net/http" "os" "os/signal" "reflect" "syscall" "time" "github.com/SlyMarbo/rss" "github.com/robfig/cron" ) // RSSEntry describes a single RSS feed and the corresponding hook type RSSEntry struct { // The URL of the RSS/Atom feed Url string `json:"url,omitempty"` // The retry count Retry int `json:"retry,omitempty"` // The hook end-point Hook string `json:"hook,omitempty"` } // Loaded contains the loaded feeds + hooks, as read from the specified // configuration file var Loaded []RSSEntry // Timeout is the (global) timeout we use when loading remote RSS // feeds. var Timeout time.Duration // loadConfig loads the named configuration file and populates our // `Loaded` list of RSS-feeds & Webhook addresses func loadConfig(filename string) { file, err := os.Open(filename) if err != nil { log.Fatalf("loadConfig: Error opening %s - %s\n", filename, err.Error()) } defer file.Close() err = json.NewDecoder(file).Decode(&Loaded) if err != nil { log.Fatalf("loadConfig: Error decoding %s - %s\n", filename, err.Error()) } log.Printf("Loaded %d feeds\n", len(Loaded)) } // checkFeeds is our work-horse. // // For each available feed it looks for new entries, and when founds // triggers `notify` upon the resulting entry func checkFeeds(feeds []rss.Feed) { // // For each thing we're monitoring // for i, feed := range feeds { // skip empty feed, probably because: // 1. feed initialization failed // 2. retry count exceeded if reflect.ValueOf(feed).IsZero() { continue } // Fetch the feed-contents err := feed.Update() if err != nil { log.Printf("checkFeeds: Error fetching %s - %s\n", Loaded[i].Url, err.Error()) if Loaded[i].Retry--; Loaded[i].Retry == 0 { log.Printf("checkFeeds: Temporally diable feed - %s\n", Loaded[i].Url) feeds[i] = rss.Feed{} } // log.Printf("checkFeeds: refresh time - %s\n", feed.Refresh.Format(time.RFC3339)) continue } if feed.Unread != 0 { notify(i, feed.Items[len(feed.Items)-1]) feed.Unread = 0 } } } // notify actually submits the specified item to the remote webhook. // The RSS-item is submitted as a JSON-object. func notify(i int, item *rss.Item) { // We'll post the item as a JSON object. // So first of all encode it. jsonValue, err := json.Marshal(item) if err != nil { log.Fatalf("notify: Failed to encode JSON:%s\n", err.Error()) } // // Post to the specified hook URL. // res, err := http.Post(Loaded[i].Hook, "application/json", bytes.NewBuffer(jsonValue)) if err != nil { log.Printf("notify: Failed to POST to %s - %s\n", Loaded[i].Hook, err.Error()) // TODO: retry? return } // // OK now we've submitted the post. // // We should retrieve the status-code + body, if the status-code // is "odd" then we'll show them. // defer res.Body.Close() _, err = io.ReadAll(res.Body) if err != nil { log.Printf("notify: Failed to read response from %s - %s\n", Loaded[i].Hook, err.Error()) return } status := res.StatusCode if status != 200 { log.Printf("notify: Warning - Status code was not 200: %d\n", status) } } // main is our entry-point func main() { // Parse the command-line flags config := flag.String("config", "config.json", "The path to the configuration-file to read") timeout := flag.Duration("timeout", 5*time.Second, "The timeout used for fetching the remote feeds") schedule := flag.String("schedule", "@every 5m", "The cron schedule for fetching the remote feeds") startup := flag.Bool("startup", false, "Run on startup (first check)") flag.Parse() // Setup the default timeout and TTL // Remember that we respect spec, like `ttl` field in RSS 2.0 rss.DefaultRefreshInterval = 10 * time.Second rss.DefaultFetchFunc = func(url string) (*http.Response, error) { client := &http.Client{Timeout: *timeout} return client.Get(url) } // // Load the configuration file // loadConfig(*config) // // Show the things we're monitoring // // for _, ent := range Loaded { // log.Printf("Monitoring feed %s\nPosting to %s\n\n", // ent.Feed, ent.Hook) // } // // Make the initial scan of feeds immediately to avoid waiting too // long for the first time. // feeds := make([]rss.Feed, len(Loaded)) for i, ent := range Loaded { go func(i int, ent RSSEntry) { feed, err := rss.Fetch(ent.Url) if err != nil { log.Printf("main: Error fetching %s - %s\n", ent.Url, err.Error()) return } // TODO: URL validiy checks if !*startup { feed.Unread = 0 } feeds[i] = *feed }(i, ent) } // // Now repeat that every five minutes or in custom schedule. // c := cron.New() c.AddFunc(*schedule, func() { checkFeeds(feeds) }) c.Start() // // Now we can loop waiting to be terminated via ctrl-c, etc. // sigs := make(chan os.Signal, 1) done := make(chan bool, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { <-sigs done <- true }() <-done }