|
|
@@ -1,40 +1,41 @@
|
|
|
// rss2hook is a simple utility which will make HTTP POST
|
|
|
-// requests to remote web-hooks when new items appear in an RSS feed.
|
|
|
+// requests to remote web-hooks or execute commands when
|
|
|
+// new items appear in an RSS feed.
|
|
|
//
|
|
|
// Steve
|
|
|
+// poesty
|
|
|
//
|
|
|
|
|
|
package main
|
|
|
|
|
|
import (
|
|
|
- "bufio"
|
|
|
"bytes"
|
|
|
- "crypto/sha1"
|
|
|
- "encoding/hex"
|
|
|
"encoding/json"
|
|
|
"flag"
|
|
|
- "fmt"
|
|
|
- "io/ioutil"
|
|
|
+ "io"
|
|
|
+ "log"
|
|
|
"net/http"
|
|
|
"os"
|
|
|
+ "os/exec"
|
|
|
"os/signal"
|
|
|
- "regexp"
|
|
|
- "strings"
|
|
|
+ "reflect"
|
|
|
"syscall"
|
|
|
"time"
|
|
|
|
|
|
- "github.com/mmcdole/gofeed"
|
|
|
+ "github.com/SlyMarbo/rss"
|
|
|
"github.com/robfig/cron"
|
|
|
)
|
|
|
|
|
|
// RSSEntry describes a single RSS feed and the corresponding hook
|
|
|
-// to POST to.
|
|
|
type RSSEntry struct {
|
|
|
// The URL of the RSS/Atom feed
|
|
|
- feed string
|
|
|
-
|
|
|
- // The end-point to make the webhook request to.
|
|
|
- hook string
|
|
|
+ Url string `json:"url,omitempty"`
|
|
|
+ // The hook method: webhook/command
|
|
|
+ Method string `json:"method,omitempty"`
|
|
|
+ // The retry count
|
|
|
+ Retry int `json:"retry,omitempty"`
|
|
|
+ // The hook end-point
|
|
|
+ Hook string `json:"hook,omitempty"`
|
|
|
}
|
|
|
|
|
|
// Loaded contains the loaded feeds + hooks, as read from the specified
|
|
|
@@ -50,223 +51,124 @@ var Timeout time.Duration
|
|
|
func loadConfig(filename string) {
|
|
|
file, err := os.Open(filename)
|
|
|
if err != nil {
|
|
|
- fmt.Printf("Error opening %s - %s\n", filename, err.Error())
|
|
|
- return
|
|
|
+ log.Fatalf("loadConfig: Error opening %s - %s\n", filename, err.Error())
|
|
|
}
|
|
|
defer file.Close()
|
|
|
|
|
|
- //
|
|
|
- // Process it line by line.
|
|
|
- //
|
|
|
- scanner := bufio.NewScanner(file)
|
|
|
- for scanner.Scan() {
|
|
|
-
|
|
|
- // Get the next line, and strip leading/trailing space
|
|
|
- tmp := scanner.Text()
|
|
|
- tmp = strings.TrimSpace(tmp)
|
|
|
-
|
|
|
- //
|
|
|
- // Skip lines that begin with a comment.
|
|
|
- //
|
|
|
- if (tmp != "") && (!strings.HasPrefix(tmp, "#")) {
|
|
|
-
|
|
|
- //
|
|
|
- // Otherwise find the feed + post-point
|
|
|
- //
|
|
|
- parser := regexp.MustCompile("^(.+?)=([^=].+)")
|
|
|
- match := parser.FindStringSubmatch(tmp)
|
|
|
-
|
|
|
- //
|
|
|
- // OK we found a suitable entry.
|
|
|
- //
|
|
|
- if len(match) == 3 {
|
|
|
-
|
|
|
- feed := strings.TrimSpace(match[1])
|
|
|
- hook := strings.TrimSpace(match[2])
|
|
|
-
|
|
|
- // Append the new entry to our list
|
|
|
- entry := RSSEntry{feed: feed, hook: hook}
|
|
|
- Loaded = append(Loaded, entry)
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-}
|
|
|
-
|
|
|
-// fetchFeed fetches the contents of the specified URL.
|
|
|
-func fetchFeed(url string) (string, error) {
|
|
|
-
|
|
|
- // Ensure we setup a timeout for our fetch
|
|
|
- client := &http.Client{Timeout: Timeout}
|
|
|
-
|
|
|
- // We'll only make a GET request
|
|
|
- req, err := http.NewRequest("GET", url, nil)
|
|
|
- if err != nil {
|
|
|
- return "", err
|
|
|
- }
|
|
|
-
|
|
|
- // We ensure we identify ourself.
|
|
|
- req.Header.Set("User-Agent", "rss2email (https://github.com/skx/rss2email)")
|
|
|
-
|
|
|
- // Make the request
|
|
|
- resp, err := client.Do(req)
|
|
|
+ err = json.NewDecoder(file).Decode(&Loaded)
|
|
|
if err != nil {
|
|
|
- return "", err
|
|
|
+ log.Fatalf("loadConfig: Error decoding %s - %s\n", filename, err.Error())
|
|
|
}
|
|
|
- defer resp.Body.Close()
|
|
|
-
|
|
|
- // Read the body returned
|
|
|
- output, err := ioutil.ReadAll(resp.Body)
|
|
|
- if err != nil {
|
|
|
- return "", err
|
|
|
- }
|
|
|
- return string(output), nil
|
|
|
-}
|
|
|
-
|
|
|
-// isNew returns TRUE if this feed-item hasn't been notified about
|
|
|
-// previously.
|
|
|
-func isNew(parent string, item *gofeed.Item) bool {
|
|
|
-
|
|
|
- hasher := sha1.New()
|
|
|
- hasher.Write([]byte(parent))
|
|
|
- hasher.Write([]byte(item.GUID))
|
|
|
- hashBytes := hasher.Sum(nil)
|
|
|
-
|
|
|
- // Hexadecimal conversion
|
|
|
- hexSha1 := hex.EncodeToString(hashBytes)
|
|
|
-
|
|
|
- if _, err := os.Stat(os.Getenv("HOME") + "/.rss2hook/seen/" + hexSha1); os.IsNotExist(err) {
|
|
|
- return true
|
|
|
- }
|
|
|
- return false
|
|
|
-}
|
|
|
-
|
|
|
-// recordSeen ensures that we won't re-announce a given feed-item.
|
|
|
-func recordSeen(parent string, item *gofeed.Item) {
|
|
|
-
|
|
|
- hasher := sha1.New()
|
|
|
- hasher.Write([]byte(parent))
|
|
|
- hasher.Write([]byte(item.GUID))
|
|
|
- hashBytes := hasher.Sum(nil)
|
|
|
-
|
|
|
- // Hexadecimal conversion
|
|
|
- hexSha1 := hex.EncodeToString(hashBytes)
|
|
|
-
|
|
|
- dir := os.Getenv("HOME") + "/.rss2hook/seen"
|
|
|
- os.MkdirAll(dir, os.ModePerm)
|
|
|
-
|
|
|
- _ = ioutil.WriteFile(dir+"/"+hexSha1, []byte(item.Link), 0644)
|
|
|
|
|
|
+ log.Printf("Loaded %d feeds\n", len(Loaded))
|
|
|
}
|
|
|
|
|
|
// checkFeeds is our work-horse.
|
|
|
//
|
|
|
// For each available feed it looks for new entries, and when founds
|
|
|
// triggers `notify` upon the resulting entry
|
|
|
-func checkFeeds() {
|
|
|
-
|
|
|
+func checkFeeds(feeds []rss.Feed) {
|
|
|
//
|
|
|
// For each thing we're monitoring
|
|
|
//
|
|
|
- for _, monitor := range Loaded {
|
|
|
-
|
|
|
- // Fetch the feed-contents
|
|
|
- content, err := fetchFeed(monitor.feed)
|
|
|
+ for i, feed := range feeds {
|
|
|
|
|
|
- if err != nil {
|
|
|
- fmt.Printf("Error fetching %s - %s\n",
|
|
|
- monitor.feed, err.Error())
|
|
|
+ // skip empty feed, probably because:
|
|
|
+ // 1. feed initialization failed
|
|
|
+ // 2. retry count exceeded
|
|
|
+ if reflect.ValueOf(feed).IsZero() {
|
|
|
continue
|
|
|
}
|
|
|
|
|
|
- // Now parse the feed contents into a set of items
|
|
|
- fp := gofeed.NewParser()
|
|
|
- feed, err := fp.ParseString(content)
|
|
|
- if err != nil {
|
|
|
- fmt.Printf("Error parsing %s contents: %s\n", monitor.feed, err.Error())
|
|
|
- continue
|
|
|
- }
|
|
|
+ // Fetch the feed-contents
|
|
|
+ err := feed.Update()
|
|
|
|
|
|
- // For each entry in the feed
|
|
|
- for _, i := range feed.Items {
|
|
|
+ if err != nil {
|
|
|
+ log.Printf("checkFeeds: Error fetching %s - %s\n",
|
|
|
+ Loaded[i].Url, err.Error())
|
|
|
|
|
|
- // If we've not already notified about this one.
|
|
|
- if isNew(monitor.feed, i) {
|
|
|
+ if Loaded[i].Retry--; Loaded[i].Retry == 0 {
|
|
|
+ log.Printf("checkFeeds: Temporally diable feed - %s\n", Loaded[i].Url)
|
|
|
+ feeds[i] = rss.Feed{}
|
|
|
+ }
|
|
|
+ // log.Printf("checkFeeds: refresh time - %s\n", feed.Refresh.Format(time.RFC3339))
|
|
|
|
|
|
- // Trigger the notification
|
|
|
- err := notify(monitor.hook, i)
|
|
|
+ continue
|
|
|
+ }
|
|
|
|
|
|
- // and if that notification succeeded
|
|
|
- // then record this item as having been
|
|
|
- // processed successfully.
|
|
|
- if err == nil {
|
|
|
- recordSeen(monitor.feed, i)
|
|
|
- }
|
|
|
- }
|
|
|
+ if feed.Unread != 0 {
|
|
|
+ notify(i, feed.Items[0])
|
|
|
+ feed.Unread = 0
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-// notify actually submits the specified item to the remote webhook.
|
|
|
-//
|
|
|
+// notify submits the specified item to the remote webhook
|
|
|
+// or execute custom commands.
|
|
|
// The RSS-item is submitted as a JSON-object.
|
|
|
-func notify(hook string, item *gofeed.Item) error {
|
|
|
-
|
|
|
- // We'll post the item as a JSON object.
|
|
|
- // So first of all encode it.
|
|
|
- jsonValue, err := json.Marshal(item)
|
|
|
- if err != nil {
|
|
|
- fmt.Printf("notify: Failed to encode JSON:%s\n", err.Error())
|
|
|
- return err
|
|
|
- }
|
|
|
+func notify(i int, item *rss.Item) {
|
|
|
+ if Loaded[i].Method == "command" {
|
|
|
+ cmd := exec.Command(Loaded[i].Hook)
|
|
|
+ err := cmd.Start()
|
|
|
+ if err != nil {
|
|
|
+ log.Fatalf("notify: Failed to start command: %s\n", err.Error())
|
|
|
+ }
|
|
|
+ } else if Loaded[i].Method == "webhook" {
|
|
|
+ // We'll post the item as a JSON object.
|
|
|
+ // So first of all encode it.
|
|
|
+ jsonValue, err := json.Marshal(item)
|
|
|
+ if err != nil {
|
|
|
+ log.Fatalf("notify: Failed to encode JSON:%s\n", err.Error())
|
|
|
+ }
|
|
|
|
|
|
- //
|
|
|
- // Post to the specified hook URL.
|
|
|
- //
|
|
|
- res, err := http.Post(hook,
|
|
|
- "application/json",
|
|
|
- bytes.NewBuffer(jsonValue))
|
|
|
+ //
|
|
|
+ // Post to the specified hook URL.
|
|
|
+ //
|
|
|
+ res, err := http.Post(Loaded[i].Hook,
|
|
|
+ "application/json",
|
|
|
+ bytes.NewBuffer(jsonValue))
|
|
|
|
|
|
- if err != nil {
|
|
|
- fmt.Printf("notify: Failed to POST to %s - %s\n",
|
|
|
- hook, err.Error())
|
|
|
- return err
|
|
|
- }
|
|
|
+ if err != nil {
|
|
|
+ log.Printf("notify: Failed to POST to %s - %s\n",
|
|
|
+ Loaded[i].Hook, err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
|
|
|
- //
|
|
|
- // OK now we've submitted the post.
|
|
|
- //
|
|
|
- // We should retrieve the status-code + body, if the status-code
|
|
|
- // is "odd" then we'll show them.
|
|
|
- //
|
|
|
- defer res.Body.Close()
|
|
|
- _, err = ioutil.ReadAll(res.Body)
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
- status := res.StatusCode
|
|
|
+ //
|
|
|
+ // OK now we've submitted the post.
|
|
|
+ //
|
|
|
+ // We should retrieve the status-code + body, if the status-code
|
|
|
+ // is "odd" then we'll show them.
|
|
|
+ //
|
|
|
+ defer res.Body.Close()
|
|
|
+ _, err = io.ReadAll(res.Body)
|
|
|
+ if err != nil {
|
|
|
+ log.Printf("notify: Failed to read response from %s - %s\n",
|
|
|
+ Loaded[i].Hook, err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ status := res.StatusCode
|
|
|
|
|
|
- if status != 200 {
|
|
|
- fmt.Printf("notify: Warning - Status code was not 200: %d\n", status)
|
|
|
+ if status != 200 {
|
|
|
+ log.Printf("notify: Warning - Status code was not 200: %d\n", status)
|
|
|
+ }
|
|
|
}
|
|
|
- return nil
|
|
|
}
|
|
|
|
|
|
// main is our entry-point
|
|
|
func main() {
|
|
|
|
|
|
// Parse the command-line flags
|
|
|
- config := flag.String("config", "", "The path to the configuration-file to read")
|
|
|
+ config := flag.String("config", "config.json", "The path to the configuration-file to read")
|
|
|
timeout := flag.Duration("timeout", 5*time.Second, "The timeout used for fetching the remote feeds")
|
|
|
+ schedule := flag.String("schedule", "@every 5m", "The cron schedule for fetching the remote feeds")
|
|
|
flag.Parse()
|
|
|
|
|
|
- // Setup the default timeout.
|
|
|
- Timeout = *timeout
|
|
|
-
|
|
|
- if *config == "" {
|
|
|
- fmt.Printf("Please specify a configuration-file to read\n")
|
|
|
- return
|
|
|
+ // Setup the default timeout and TTL
|
|
|
+ rss.DefaultRefreshInterval = 10 * time.Second
|
|
|
+ rss.DefaultFetchFunc = func(url string) (*http.Response, error) {
|
|
|
+ client := &http.Client{Timeout: *timeout}
|
|
|
+ return client.Get(url)
|
|
|
}
|
|
|
|
|
|
//
|
|
|
@@ -277,22 +179,34 @@ func main() {
|
|
|
//
|
|
|
// Show the things we're monitoring
|
|
|
//
|
|
|
- for _, ent := range Loaded {
|
|
|
- fmt.Printf("Monitoring feed %s\nPosting to %s\n\n",
|
|
|
- ent.feed, ent.hook)
|
|
|
- }
|
|
|
+ // for _, ent := range Loaded {
|
|
|
+ // log.Printf("Monitoring feed %s\nPosting to %s\n\n",
|
|
|
+ // ent.Feed, ent.Hook)
|
|
|
+ // }
|
|
|
|
|
|
//
|
|
|
// Make the initial scan of feeds immediately to avoid waiting too
|
|
|
// long for the first time.
|
|
|
//
|
|
|
- checkFeeds()
|
|
|
+ feeds := make([]rss.Feed, len(Loaded))
|
|
|
+ for i, ent := range Loaded {
|
|
|
+ go func(i int, ent RSSEntry) {
|
|
|
+ feed, err := rss.Fetch(ent.Url)
|
|
|
+ if err != nil {
|
|
|
+ log.Printf("main: Error fetching %s - %s\n",
|
|
|
+ ent.Url, err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // feed.Unread = 0
|
|
|
+ feeds[i] = *feed
|
|
|
+ }(i, ent)
|
|
|
+ }
|
|
|
|
|
|
//
|
|
|
- // Now repeat that every five minutes.
|
|
|
+ // Now repeat that every five minutes or in custom schedule.
|
|
|
//
|
|
|
c := cron.New()
|
|
|
- c.AddFunc("@every 5m", func() { checkFeeds() })
|
|
|
+ c.AddFunc(*schedule, func() { checkFeeds(feeds) })
|
|
|
c.Start()
|
|
|
|
|
|
//
|
