// rss2hook is a simple utility which will make HTTP POST // requests to remote web-hooks when new items appear in an RSS feed. // // Steve // package main import ( "bufio" "bytes" "crypto/sha1" "encoding/hex" "encoding/json" "flag" "fmt" "io/ioutil" "net/http" "os" "os/signal" "regexp" "strings" "syscall" "time" "github.com/mmcdole/gofeed" "github.com/robfig/cron" ) // RSSEntry describes a single RSS feed and the corresponding hook // to POST to. type RSSEntry struct { // The URL of the RSS/Atom feed feed string // The end-point to make the webhook request to. hook string } // Loaded contains the loaded feeds + hooks, as read from the specified // configuration file var Loaded []RSSEntry // Timeout is the (global) timeout we use when loading remote RSS // feeds. var Timeout time.Duration // loadConfig loads the named configuration file and populates our // `Loaded` list of RSS-feeds & Webhook addresses func loadConfig(filename string) { file, err := os.Open(filename) if err != nil { fmt.Printf("Error opening %s - %s\n", filename, err.Error()) return } defer file.Close() // // Process it line by line. // scanner := bufio.NewScanner(file) for scanner.Scan() { // Get the next line, and strip leading/trailing space tmp := scanner.Text() tmp = strings.TrimSpace(tmp) // // Skip lines that begin with a comment. // if (tmp != "") && (!strings.HasPrefix(tmp, "#")) { // // Otherwise find the feed + post-point // parser := regexp.MustCompile("^(.+?)=([^=].+)") match := parser.FindStringSubmatch(tmp) // // OK we found a suitable entry. // if len(match) == 3 { feed := strings.TrimSpace(match[1]) hook := strings.TrimSpace(match[2]) // Append the new entry to our list entry := RSSEntry{feed: feed, hook: hook} Loaded = append(Loaded, entry) } } } } // fetchFeed fetches the contents of the specified URL. func fetchFeed(url string) (string, error) { // Ensure we setup a timeout for our fetch client := &http.Client{Timeout: Timeout} // We'll only make a GET request req, err := http.NewRequest("GET", url, nil) if err != nil { return "", err } // We ensure we identify ourself. req.Header.Set("User-Agent", "rss2email (https://github.com/skx/rss2email)") // Make the request resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() // Read the body returned output, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err } return string(output), nil } // isNew returns TRUE if this feed-item hasn't been notified about // previously. func isNew(parent string, item *gofeed.Item) bool { hasher := sha1.New() hasher.Write([]byte(parent)) hasher.Write([]byte(item.GUID)) hashBytes := hasher.Sum(nil) // Hexadecimal conversion hexSha1 := hex.EncodeToString(hashBytes) if _, err := os.Stat(os.Getenv("HOME") + "/.rss2hook/seen/" + hexSha1); os.IsNotExist(err) { return true } return false } // recordSeen ensures that we won't re-announce a given feed-item. func recordSeen(parent string, item *gofeed.Item) { hasher := sha1.New() hasher.Write([]byte(parent)) hasher.Write([]byte(item.GUID)) hashBytes := hasher.Sum(nil) // Hexadecimal conversion hexSha1 := hex.EncodeToString(hashBytes) dir := os.Getenv("HOME") + "/.rss2hook/seen" os.MkdirAll(dir, os.ModePerm) _ = ioutil.WriteFile(dir+"/"+hexSha1, []byte(item.Link), 0644) } // checkFeeds is our work-horse. // // For each available feed it looks for new entries, and when founds // triggers `notify` upon the resulting entry func checkFeeds() { // // For each thing we're monitoring // for _, monitor := range Loaded { // Fetch the feed-contents content, err := fetchFeed(monitor.feed) if err != nil { fmt.Printf("Error fetching %s - %s\n", monitor.feed, err.Error()) continue } // Now parse the feed contents into a set of items fp := gofeed.NewParser() feed, err := fp.ParseString(content) if err != nil { fmt.Printf("Error parsing %s contents: %s\n", monitor.feed, err.Error()) continue } // For each entry in the feed for _, i := range feed.Items { // If we've not already notified about this one. if isNew(monitor.feed, i) { // Trigger the notification err := notify(monitor.hook, i) // and if that notification succeeded // then record this item as having been // processed successfully. if err == nil { recordSeen(monitor.feed, i) } } } } } // notify actually submits the specified item to the remote webhook. // // The RSS-item is submitted as a JSON-object. func notify(hook string, item *gofeed.Item) error { // We'll post the item as a JSON object. // So first of all encode it. jsonValue, err := json.Marshal(item) if err != nil { fmt.Printf("notify: Failed to encode JSON:%s\n", err.Error()) return err } // // Post to the specified hook URL. // res, err := http.Post(hook, "application/json", bytes.NewBuffer(jsonValue)) if err != nil { fmt.Printf("notify: Failed to POST to %s - %s\n", hook, err.Error()) return err } // // OK now we've submitted the post. // // We should retrieve the status-code + body, if the status-code // is "odd" then we'll show them. // defer res.Body.Close() _, err = ioutil.ReadAll(res.Body) if err != nil { return err } status := res.StatusCode if status != 200 { fmt.Printf("notify: Warning - Status code was not 200: %d\n", status) } return nil } // main is our entry-point func main() { // Parse the command-line flags config := flag.String("config", "", "The path to the configuration-file to read") timeout := flag.Duration("timeout", 5*time.Second, "The timeout used for fetching the remote feeds") flag.Parse() // Setup the default timeout. Timeout = *timeout if *config == "" { fmt.Printf("Please specify a configuration-file to read\n") return } // // Load the configuration file // loadConfig(*config) // // Show the things we're monitoring // for _, ent := range Loaded { fmt.Printf("Monitoring feed %s\nPosting to %s\n\n", ent.feed, ent.hook) } // // Make the initial scan of feeds immediately to avoid waiting too // long for the first time. // checkFeeds() // // Now repeat that every five minutes. // c := cron.New() c.AddFunc("@every 5m", func() { checkFeeds() }) c.Start() // // Now we can loop waiting to be terminated via ctrl-c, etc. // sigs := make(chan os.Signal, 1) done := make(chan bool, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { <-sigs done <- true }() <-done }