From f1ffa6e1e3c9083781be26d224932d7832938133 Mon Sep 17 00:00:00 2001 From: emersion Date: Sat, 13 Aug 2016 12:10:19 +0200 Subject: [PATCH 01/12] Adds filesystem storage --- fs.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 fs.go diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..a026af7 --- /dev/null +++ b/fs.go @@ -0,0 +1,54 @@ +package checkup + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "time" +) + +// FS is a way to store checkup results on the local filesystem. +type FS struct { + Dir string `json:"dir"` + Url string `json:"url"` + + // Check files older than CheckExpiry will be + // deleted on calls to Maintain(). If this is + // the zero value, no old check files will be + // deleted. + CheckExpiry time.Duration `json:"check_expiry,omitempty"` +} + +// Store stores results on filesystem according to the configuration in fs. +func (fs FS) Store(results []Result) error { + f, err := os.Create(filepath.Join(fs.Dir, *GenerateFilename())) + if err != nil { + return err + } + defer f.Close() + + return json.NewEncoder(f).Encode(results) +} + +// Maintain deletes check files that are older than fs.CheckExpiry. +func (fs FS) Maintain() error { + if fs.CheckExpiry == 0 { + return nil + } + + files, err := ioutil.ReadDir(fs.Dir) + if err != nil { + return err + } + + for _, f := range files { + if time.Since(f.ModTime()) > fs.CheckExpiry { + if err := os.Remove(f.Name()); err != nil { + return err + } + } + } + + return nil +} From e26f542d0138870464e63747fc5eecc545e41dc6 Mon Sep 17 00:00:00 2001 From: emersion Date: Sat, 13 Aug 2016 14:37:09 +0200 Subject: [PATCH 02/12] Adds FS to checkup.go --- checkup.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/checkup.go b/checkup.go index ef69e08..424e476 100644 --- a/checkup.go +++ b/checkup.go @@ -200,6 +200,8 @@ func (c Checkup) MarshalJSON() ([]byte, error) { switch c.Storage.(type) { case S3: providerName = "s3" + case FS: + providerName = "fs" default: return result, fmt.Errorf("unknown Storage type") } @@ -297,6 +299,13 @@ func (c *Checkup) UnmarshalJSON(b []byte) error { return err } c.Storage = storage + case "fs": + var storage FS + err = json.Unmarshal(raw.Storage, &storage) + if err != nil { + return err + } + c.Storage = storage default: return fmt.Errorf("%s: unknown Storage type", types.Storage.Provider) } From e01ad88422ff1dd27234fd7f4692d2789f5eaa38 Mon Sep 17 00:00:00 2001 From: emersion Date: Sat, 13 Aug 2016 15:26:29 +0200 Subject: [PATCH 03/12] Adds index file to FS --- fs.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/fs.go b/fs.go index a026af7..115f029 100644 --- a/fs.go +++ b/fs.go @@ -8,9 +8,13 @@ import ( "time" ) +const indexName = "index.json" + // FS is a way to store checkup results on the local filesystem. type FS struct { + // The path to the directory where check files will be stored. Dir string `json:"dir"` + // The URL corresponding to fs.Dir. Url string `json:"url"` // Check files older than CheckExpiry will be @@ -20,15 +24,52 @@ type FS struct { CheckExpiry time.Duration `json:"check_expiry,omitempty"` } +func (fs FS) readIndex() (index map[string]int64, err error) { + f, err := os.Open(filepath.Join(fs.Dir, indexName)) + if err != nil { + return + } + defer f.Close() + + err = json.NewDecoder(f).Decode(&index) + return +} + +func (fs FS) writeIndex(index map[string]int64) error { + f, err := os.Create(filepath.Join(fs.Dir, indexName)) + if err != nil { + return err + } + defer f.Close() + + return json.NewEncoder(f).Encode(index) +} + // Store stores results on filesystem according to the configuration in fs. func (fs FS) Store(results []Result) error { - f, err := os.Create(filepath.Join(fs.Dir, *GenerateFilename())) + // Write results to a new file + name := *GenerateFilename() + f, err := os.Create(filepath.Join(fs.Dir, name)) + if err != nil { + return err + } + err = json.NewEncoder(f).Encode(results) + f.Close() + if err != nil { + return err + } + + // Read current index file + index, err := fs.readIndex() if err != nil { return err } - defer f.Close() - return json.NewEncoder(f).Encode(results) + // Add new file to index + index[name] = time.Now().UnixNano() + + // Write new index + return fs.writeIndex(index) } // Maintain deletes check files that are older than fs.CheckExpiry. @@ -42,13 +83,25 @@ func (fs FS) Maintain() error { return err } + index, err := fs.readIndex() + if err != nil { + return err + } + for _, f := range files { - if time.Since(f.ModTime()) > fs.CheckExpiry { + name, _ := filepath.Rel(fs.Dir, f.Name()) + if name == indexName { + continue + } + + nsec, ok := index[name] + if !ok || time.Since(time.Unix(0, nsec)) > fs.CheckExpiry { if err := os.Remove(f.Name()); err != nil { return err } + delete(index, name) } } - return nil + return fs.writeIndex(index) } From e6d4114121cb7cc6697d5b6b9ce4385cd0ce322d Mon Sep 17 00:00:00 2001 From: emersion Date: Sat, 13 Aug 2016 15:29:48 +0200 Subject: [PATCH 04/12] Adds error handling to filepath.Rel() in FS --- fs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs.go b/fs.go index 115f029..aef33c2 100644 --- a/fs.go +++ b/fs.go @@ -89,8 +89,8 @@ func (fs FS) Maintain() error { } for _, f := range files { - name, _ := filepath.Rel(fs.Dir, f.Name()) - if name == indexName { + name, err := filepath.Rel(fs.Dir, f.Name()) + if err != nil || name == indexName { continue } From 0e3d3aa6618c5883fbe662a4450fa9d40a0c4de9 Mon Sep 17 00:00:00 2001 From: emersion Date: Sat, 13 Aug 2016 15:48:11 +0200 Subject: [PATCH 05/12] Adds FS JS storage --- fs.go | 6 +++- statuspage/js/fs.js | 68 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 statuspage/js/fs.js diff --git a/fs.go b/fs.go index aef33c2..ec94b15 100644 --- a/fs.go +++ b/fs.go @@ -95,7 +95,11 @@ func (fs FS) Maintain() error { } nsec, ok := index[name] - if !ok || time.Since(time.Unix(0, nsec)) > fs.CheckExpiry { + if !ok { + continue + } + + if time.Since(time.Unix(0, nsec)) > fs.CheckExpiry { if err := os.Remove(f.Name()); err != nil { return err } diff --git a/statuspage/js/fs.js b/statuspage/js/fs.js new file mode 100644 index 0000000..0215ca7 --- /dev/null +++ b/statuspage/js/fs.js @@ -0,0 +1,68 @@ +/** + +FS Storage Adapter for Checkup.js + +**/ + +var checkup = checkup || {}; + +checkup.storage = (function() { + var url; + + // getCheckFileList gets the list of check files within + // the given timeframe (as a unit of nanoseconds) to + // download. + function getCheckFileList(timeframe, callback) { + var after = time.Now() - timeframe; + checkup.getJSON(url+'/index.json', function(index) { + var names = []; + for (var name in index) { + if (index[name] >= after) { + names.push(name); + } + } + callback(names); + }); + }; + + // setup prepares this storage unit to operate. + this.setup = function(cfg) { + url = cfg.url; + }; + + // getChecksWithin gets all the checks within timeframe as a unit + // of nanoseconds, and executes callback for each check file. + this.getChecksWithin = function(timeframe, fileCallback, doneCallback) { + var checksLoaded = 0, resultsLoaded = 0; + getCheckFileList(timeframe, function(list) { + if (list.length == 0 && (typeof doneCallback === 'function')) { + doneCallback(checksLoaded); + } else { + for (var i = 0; i < list.length; i++) { + checkup.getJSON(url+'/'+list[i], function(filename) { + return function(json, url) { + checksLoaded++; + resultsLoaded += json.length; + if (typeof fileCallback === 'function') + fileCallback(json, filename); + if (checksLoaded >= list.length && (typeof doneCallback === 'function')) + doneCallback(checksLoaded, resultsLoaded); + }; + }(list[i])); + } + } + }); + }; + + // getNewChecks gets any checks since the timestamp on the file name + // of the youngest check file that has been downloaded. If no check + // files have been downloaded, no new check files will be loaded. + this.getNewChecks = function(fileCallback, doneCallback) { + if (!checkup.lastCheckTs == null) + return; + var timeframe = time.Now() - checkup.lastCheckTs; + return this.getChecksWithin(timeframe, fileCallback, doneCallback); + }; + + return this; +})(); From 7590cf766178d755a85c0c0f609e29af5880c4c2 Mon Sep 17 00:00:00 2001 From: emersion Date: Sat, 13 Aug 2016 16:00:23 +0200 Subject: [PATCH 06/12] fs: do not fail if index doesn't exist --- fs.go | 5 +++++ statuspage/index.html | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/fs.go b/fs.go index ec94b15..f797a61 100644 --- a/fs.go +++ b/fs.go @@ -26,6 +26,11 @@ type FS struct { func (fs FS) readIndex() (index map[string]int64, err error) { f, err := os.Open(filepath.Join(fs.Dir, indexName)) + if os.IsNotExist(err) { + index = map[string]int64{} + err = nil + return + } if err != nil { return } diff --git a/statuspage/index.html b/statuspage/index.html index a8979b4..2e25ecd 100644 --- a/statuspage/index.html +++ b/statuspage/index.html @@ -5,7 +5,7 @@ - + From 56ed22a73ac265aed93f484d38f322a99bf05a81 Mon Sep 17 00:00:00 2001 From: emersion Date: Sun, 14 Aug 2016 15:51:25 +0200 Subject: [PATCH 07/12] Adds FS tests for Storage --- fs_test.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 fs_test.go diff --git a/fs_test.go b/fs_test.go new file mode 100644 index 0000000..984e039 --- /dev/null +++ b/fs_test.go @@ -0,0 +1,68 @@ +package checkup + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" +) + +func newTempFS() (specimen FS, err error) { + dir, err := ioutil.TempDir("", "checkup") + if err != nil { + return + } + + specimen = FS{ + Dir: dir, + } + return +} + +func TestFS_Store(t *testing.T) { + results := []Result{{Title: "Testing"}} + resultsBytes := []byte(`[{"title":"Testing"}]`+"\n") + + specimen, err := newTempFS() + if err != nil { + t.Fatalf("Cannot create temporary directory: %v", err) + } + defer os.RemoveAll(specimen.Dir) + + if err := specimen.Store(results); err != nil { + t.Fatalf("Expected no error from Store(), got: %v", err) + } + + // Make sure index has been created + index, err := specimen.readIndex() + if err != nil { + t.Fatalf("Cannot read index: %v", err) + } + + if len(index) != 1 { + t.Fatalf("Expected length of index to be 1, but got %v", len(index)) + } + + var ( + name string + nsec int64 + ) + for name, nsec = range index {} + + // Make sure index has timestamp of check + ts := time.Unix(0, nsec) + if time.Since(ts) > 1*time.Second { + t.Errorf("Timestamp of check is %s but expected something very recent", ts) + } + + // Make sure check file bytes are correct + b, err := ioutil.ReadFile(filepath.Join(specimen.Dir, name)) + if err != nil { + t.Fatalf("Expected no error reading body, got: %v", err) + } + if bytes.Compare(b, resultsBytes) != 0 { + t.Errorf("Contents of file are wrong\nExpected %s\nGot %s", resultsBytes, b) + } +} From 5551226d6ea9fc3e5c6e924f72588a1cda9179b1 Mon Sep 17 00:00:00 2001 From: emersion Date: Sun, 14 Aug 2016 15:52:54 +0200 Subject: [PATCH 08/12] Replaces Url by URL --- fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs.go b/fs.go index f797a61..38a7dee 100644 --- a/fs.go +++ b/fs.go @@ -15,7 +15,7 @@ type FS struct { // The path to the directory where check files will be stored. Dir string `json:"dir"` // The URL corresponding to fs.Dir. - Url string `json:"url"` + URL string `json:"url"` // Check files older than CheckExpiry will be // deleted on calls to Maintain(). If this is From fb3c345b23f739765a7fb17fde8e348b638beef5 Mon Sep 17 00:00:00 2001 From: emersion Date: Sun, 14 Aug 2016 18:25:01 +0200 Subject: [PATCH 09/12] Adds FS test for Maintain(), fixes bug --- fs.go | 9 ++++----- fs_test.go | 42 ++++++++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/fs.go b/fs.go index 38a7dee..24008fc 100644 --- a/fs.go +++ b/fs.go @@ -94,21 +94,20 @@ func (fs FS) Maintain() error { } for _, f := range files { - name, err := filepath.Rel(fs.Dir, f.Name()) - if err != nil || name == indexName { + if f.Name() == indexName { continue } - nsec, ok := index[name] + nsec, ok := index[f.Name()] if !ok { continue } if time.Since(time.Unix(0, nsec)) > fs.CheckExpiry { - if err := os.Remove(f.Name()); err != nil { + if err := os.Remove(filepath.Join(fs.Dir, f.Name())); err != nil { return err } - delete(index, name) + delete(index, f.Name()) } } diff --git a/fs_test.go b/fs_test.go index 984e039..2f2be28 100644 --- a/fs_test.go +++ b/fs_test.go @@ -9,27 +9,19 @@ import ( "time" ) -func newTempFS() (specimen FS, err error) { - dir, err := ioutil.TempDir("", "checkup") - if err != nil { - return - } - - specimen = FS{ - Dir: dir, - } - return -} - -func TestFS_Store(t *testing.T) { +func TestFS(t *testing.T) { results := []Result{{Title: "Testing"}} resultsBytes := []byte(`[{"title":"Testing"}]`+"\n") - specimen, err := newTempFS() + dir, err := ioutil.TempDir("", "checkup") if err != nil { t.Fatalf("Cannot create temporary directory: %v", err) } - defer os.RemoveAll(specimen.Dir) + defer os.RemoveAll(dir) + + specimen := FS{ + Dir: dir, + } if err := specimen.Store(results); err != nil { t.Fatalf("Expected no error from Store(), got: %v", err) @@ -58,11 +50,29 @@ func TestFS_Store(t *testing.T) { } // Make sure check file bytes are correct - b, err := ioutil.ReadFile(filepath.Join(specimen.Dir, name)) + checkfile := filepath.Join(specimen.Dir, name) + b, err := ioutil.ReadFile(checkfile) if err != nil { t.Fatalf("Expected no error reading body, got: %v", err) } if bytes.Compare(b, resultsBytes) != 0 { t.Errorf("Contents of file are wrong\nExpected %s\nGot %s", resultsBytes, b) } + + // Make sure check file is not deleted after maintain with CheckExpiry == 0 + if err := specimen.Maintain(); err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if _, err := os.Stat(checkfile); err != nil { + t.Fatalf("Expected not error calling Stat() on checkfile, got: %v", err) + } + + // Make sure checkfile is deleted after maintain with CheckExpiry > 0 + specimen.CheckExpiry = 1 * time.Nanosecond + if err := specimen.Maintain(); err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if _, err := os.Stat(checkfile); !os.IsNotExist(err) { + t.Fatalf("Expected checkfile to be deleted, but Stat() returned error: %v", err) + } } From 1d19e996805fe85ccf9edd96a7808aea5f685971 Mon Sep 17 00:00:00 2001 From: emersion Date: Sun, 14 Aug 2016 19:25:11 +0200 Subject: [PATCH 10/12] Updates README.md --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 26d65df..06dd94f 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Out of the box, Checkup currently supports: - Checking HTTP endpoints - Checking TCP endpoints (TLS supported) -- Storing results on S3 +- Storing results on S3 or on the local filesystem - Viewing results on a status page that is mobile-responsive and 100% static @@ -143,7 +143,7 @@ As you perform checks, the status page will update every so often with the lates ### Performing checks -You can run checks many different ways: cron, AWS Lambda, or a time.Ticker in your own Go program, to name a few. Checks should be run on a regular basis. How often you run checks depends on your requirements and how much time you render on the status page. +You can run checks many different ways: cron, AWS Lambda, or a time.Ticker in your own Go program, to name a few. Checks should be run on a regular basis. How often you run checks depends on your requirements and how much time you render on the status page. For example, if you run checks every 10 minutes, showing the last 24 hours on the status page will require 144 check files to be downloaded on each page load. You can distribute your checks to help avoid localized network problems, but this multiplies the number of files by the number of nodes you run checks on, so keep that in mind. @@ -287,4 +287,3 @@ Uh oh, having some fires? 🔥 You can create a type that implements `checkup.No #### Other kinds of checks or storage providers Need to check more than HTTP? S3 too Amazony for you? You can implement your own Checker and Storage types. If it's general enough, feel free to submit a pull request so others can use it too! - From a46d719daadb9ae969921a82d1421dc76f691168 Mon Sep 17 00:00:00 2001 From: emersion Date: Thu, 18 Aug 2016 23:49:32 +0200 Subject: [PATCH 11/12] Do not change default js storage --- statuspage/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statuspage/index.html b/statuspage/index.html index 2e25ecd..a8979b4 100644 --- a/statuspage/index.html +++ b/statuspage/index.html @@ -5,7 +5,7 @@ - + From 0ea536878d200db29b07e98c0d5b1c6b20965cb4 Mon Sep 17 00:00:00 2001 From: emersion Date: Thu, 18 Aug 2016 23:52:03 +0200 Subject: [PATCH 12/12] Style fixes to FS.readIndex() --- fs.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fs.go b/fs.go index 24008fc..ec8dddc 100644 --- a/fs.go +++ b/fs.go @@ -24,20 +24,19 @@ type FS struct { CheckExpiry time.Duration `json:"check_expiry,omitempty"` } -func (fs FS) readIndex() (index map[string]int64, err error) { +func (fs FS) readIndex() (map[string]int64, error) { + index := map[string]int64{} + f, err := os.Open(filepath.Join(fs.Dir, indexName)) if os.IsNotExist(err) { - index = map[string]int64{} - err = nil - return - } - if err != nil { - return + return index, nil + } else if err != nil { + return nil, err } defer f.Close() err = json.NewDecoder(f).Decode(&index) - return + return index, err } func (fs FS) writeIndex(index map[string]int64) error {