From 77c172b823b64ebface655681ab0749b9d2f7081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Benencia?= Date: Fri, 13 Apr 2018 16:30:31 -0700 Subject: First public commit --- .gitignore | 31 + Gopkg.lock | 66 + Gopkg.toml | 50 + LICENSE | 201 + Makefile | 15 + README.md | 199 + .../cloud-config/cloud-config-release.slc | 13 + .../cloud-config/users.slc | 11 + .../example-templates-configs/ipxe/coreos.ipxe.slc | 16 + .../example-templates-configs/ipxe/debian.ipxe.slc | 16 + .../example-templates-configs/ipxe/linux.cfg.slc | 15 + .../ipxe/storage.ipxe.slc | 16 + .../ipxe/ubuntu-minimal.ipxe.slc | 16 + configs/example-templates-configs/mappings.yaml | 17 + .../preseed/common.preseed.slc | 12 + .../preseed/debian.preseed.slc | 36 + .../preseed/storage.preseed.slc | 104 + .../preseed/ubuntu-minimal.preseed.slc | 36 + .../example-templates-configs/static/test-script | 3 + configs/shoelaces.conf | 6 + internal/environment/environment.go | 173 + internal/environment/environment_test.go | 61 + internal/environment/flags.go | 57 + internal/event/event.go | 100 + internal/event/event_test.go | 72 + internal/handlers/common.go | 75 + internal/handlers/events.go | 36 + internal/handlers/ipxemenu.go | 64 + internal/handlers/middleware.go | 110 + internal/handlers/polling.go | 161 + internal/handlers/static.go | 127 + internal/handlers/templates.go | 83 + internal/ipxe/ipxescript.go | 87 + internal/log/log.go | 61 + internal/mappings/mappings.go | 80 + internal/mappings/mappings_test.go | 100 + internal/mappings/parse.go | 78 + internal/polling/polling.go | 257 + internal/router/router.go | 63 + internal/server/server.go | 123 + internal/templates/templates.go | 233 + internal/utils/util_test.go | 39 + internal/utils/utils.go | 105 + main.go | 34 + shoelaces-overview.png | Bin 0 -> 21554 bytes test/integ-test/INTEGTEST.md | 0 .../expected-results/configs-static-default.txt | 4 + test/integ-test/expected-results/ipxemenu.txt | 12 + test/integ-test/expected-results/poll-k8s1-1.txt | 14 + test/integ-test/expected-results/poll-k8s1-2.txt | 14 + .../expected-results/poll-k8s1-3-stg.txt | 14 + .../expected-results/poll-k8s1-4-stg.txt | 14 + .../expected-results/poll-unknown-set-from-ui.txt | 14 + test/integ-test/expected-results/poll-unknown.txt | 2 + test/integ-test/expected-results/poll.txt | 2 + .../integ-test/expected-results/rc.local-bootstrap | 4 + test/integ-test/expected-results/static.html | 7 + .../env_overrides/production/ipxe/coreos.ipxe.slc | 16 + .../staging/preseed/example.preseed.slc | 14 + .../integ-test-configs/ipxe/coreos.ipxe.slc | 16 + test/integ-test/integ-test-configs/mappings.yaml | 39 + .../integ-test-configs/static/bootstrap.sh | 11 + .../integ-test-configs/static/rc.local-bootstrap | 4 + test/integ-test/integ_test.py | 216 + vendor/github.com/go-kit/kit/LICENSE | 22 + vendor/github.com/go-kit/kit/log/README.md | 147 + vendor/github.com/go-kit/kit/log/doc.go | 116 + vendor/github.com/go-kit/kit/log/json_logger.go | 89 + vendor/github.com/go-kit/kit/log/level/doc.go | 22 + vendor/github.com/go-kit/kit/log/level/level.go | 205 + vendor/github.com/go-kit/kit/log/log.go | 135 + vendor/github.com/go-kit/kit/log/logfmt_logger.go | 62 + vendor/github.com/go-kit/kit/log/nop_logger.go | 8 + vendor/github.com/go-kit/kit/log/stdlib.go | 116 + vendor/github.com/go-kit/kit/log/sync.go | 116 + vendor/github.com/go-kit/kit/log/term/LICENSE | 21 + vendor/github.com/go-kit/kit/log/value.go | 102 + vendor/github.com/go-logfmt/logfmt/.gitignore | 4 + vendor/github.com/go-logfmt/logfmt/.travis.yml | 15 + vendor/github.com/go-logfmt/logfmt/LICENSE | 22 + vendor/github.com/go-logfmt/logfmt/README.md | 33 + vendor/github.com/go-logfmt/logfmt/decode.go | 237 + vendor/github.com/go-logfmt/logfmt/doc.go | 6 + vendor/github.com/go-logfmt/logfmt/encode.go | 321 + vendor/github.com/go-logfmt/logfmt/fuzz.go | 126 + vendor/github.com/go-logfmt/logfmt/jsonstring.go | 277 + vendor/github.com/go-stack/stack/.travis.yml | 13 + vendor/github.com/go-stack/stack/LICENSE.md | 21 + vendor/github.com/go-stack/stack/README.md | 38 + vendor/github.com/go-stack/stack/stack.go | 324 + vendor/github.com/gorilla/context/.travis.yml | 19 + vendor/github.com/gorilla/context/LICENSE | 27 + vendor/github.com/gorilla/context/README.md | 7 + vendor/github.com/gorilla/context/context.go | 143 + vendor/github.com/gorilla/context/doc.go | 82 + vendor/github.com/gorilla/mux/.travis.yml | 22 + vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md | 11 + vendor/github.com/gorilla/mux/LICENSE | 27 + vendor/github.com/gorilla/mux/README.md | 560 ++ vendor/github.com/gorilla/mux/context_gorilla.go | 26 + vendor/github.com/gorilla/mux/context_native.go | 24 + vendor/github.com/gorilla/mux/doc.go | 307 + vendor/github.com/gorilla/mux/middleware.go | 28 + vendor/github.com/gorilla/mux/mux.go | 585 ++ vendor/github.com/gorilla/mux/regexp.go | 332 + vendor/github.com/gorilla/mux/route.go | 761 ++ vendor/github.com/gorilla/mux/test_helpers.go | 18 + vendor/github.com/justinas/alice/.travis.yml | 17 + vendor/github.com/justinas/alice/LICENSE | 20 + vendor/github.com/justinas/alice/README.md | 98 + vendor/github.com/justinas/alice/chain.go | 112 + vendor/github.com/kr/logfmt/.gitignore | 3 + vendor/github.com/kr/logfmt/Readme | 12 + vendor/github.com/kr/logfmt/decode.go | 184 + vendor/github.com/kr/logfmt/scanner.go | 149 + vendor/github.com/kr/logfmt/unquote.go | 149 + vendor/github.com/namsral/flag/LICENSE | 27 + vendor/github.com/namsral/flag/README.md | 200 + vendor/github.com/namsral/flag/flag.go | 1068 +++ vendor/gopkg.in/yaml.v2/.travis.yml | 12 + vendor/gopkg.in/yaml.v2/LICENSE | 201 + vendor/gopkg.in/yaml.v2/LICENSE.libyaml | 31 + vendor/gopkg.in/yaml.v2/NOTICE | 13 + vendor/gopkg.in/yaml.v2/README.md | 133 + vendor/gopkg.in/yaml.v2/apic.go | 739 ++ vendor/gopkg.in/yaml.v2/decode.go | 775 ++ vendor/gopkg.in/yaml.v2/emitterc.go | 1685 ++++ vendor/gopkg.in/yaml.v2/encode.go | 362 + vendor/gopkg.in/yaml.v2/go.mod | 5 + vendor/gopkg.in/yaml.v2/parserc.go | 1095 +++ vendor/gopkg.in/yaml.v2/readerc.go | 412 + vendor/gopkg.in/yaml.v2/resolve.go | 258 + vendor/gopkg.in/yaml.v2/scannerc.go | 2696 ++++++ vendor/gopkg.in/yaml.v2/sorter.go | 113 + vendor/gopkg.in/yaml.v2/writerc.go | 26 + vendor/gopkg.in/yaml.v2/yaml.go | 466 + vendor/gopkg.in/yaml.v2/yamlh.go | 738 ++ vendor/gopkg.in/yaml.v2/yamlprivateh.go | 173 + web/css/bootstrap.css | 8975 ++++++++++++++++++++ web/css/bootstrap.min.css | 7 + web/css/default.css | 148 + web/css/spinner.css | 147 + web/fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20127 bytes web/fonts/glyphicons-halflings-regular.svg | 288 + web/fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 45404 bytes web/fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes web/fonts/glyphicons-halflings-regular.woff2 | Bin 0 -> 18028 bytes web/img/favicon-32x32.png | Bin 0 -> 162795 bytes web/img/shoelaces-logo.png | Bin 0 -> 165697 bytes web/js/bootstrap.js | 3894 +++++++++ web/js/bootstrap.min.js | 7 + web/js/jquery.min.js | 6 + web/js/local.js | 117 + web/templates/html/error.html | 5 + web/templates/html/events.html | 15 + web/templates/html/footer.html | 13 + web/templates/html/header.html | 46 + web/templates/html/index.html | 67 + web/templates/html/mappings.html | 40 + 159 files changed, 34999 insertions(+) create mode 100644 .gitignore create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 configs/example-templates-configs/cloud-config/cloud-config-release.slc create mode 100644 configs/example-templates-configs/cloud-config/users.slc create mode 100644 configs/example-templates-configs/ipxe/coreos.ipxe.slc create mode 100644 configs/example-templates-configs/ipxe/debian.ipxe.slc create mode 100644 configs/example-templates-configs/ipxe/linux.cfg.slc create mode 100644 configs/example-templates-configs/ipxe/storage.ipxe.slc create mode 100644 configs/example-templates-configs/ipxe/ubuntu-minimal.ipxe.slc create mode 100644 configs/example-templates-configs/mappings.yaml create mode 100644 configs/example-templates-configs/preseed/common.preseed.slc create mode 100644 configs/example-templates-configs/preseed/debian.preseed.slc create mode 100644 configs/example-templates-configs/preseed/storage.preseed.slc create mode 100644 configs/example-templates-configs/preseed/ubuntu-minimal.preseed.slc create mode 100644 configs/example-templates-configs/static/test-script create mode 100644 configs/shoelaces.conf create mode 100644 internal/environment/environment.go create mode 100644 internal/environment/environment_test.go create mode 100644 internal/environment/flags.go create mode 100644 internal/event/event.go create mode 100644 internal/event/event_test.go create mode 100644 internal/handlers/common.go create mode 100644 internal/handlers/events.go create mode 100644 internal/handlers/ipxemenu.go create mode 100644 internal/handlers/middleware.go create mode 100644 internal/handlers/polling.go create mode 100644 internal/handlers/static.go create mode 100644 internal/handlers/templates.go create mode 100644 internal/ipxe/ipxescript.go create mode 100644 internal/log/log.go create mode 100644 internal/mappings/mappings.go create mode 100644 internal/mappings/mappings_test.go create mode 100644 internal/mappings/parse.go create mode 100644 internal/polling/polling.go create mode 100644 internal/router/router.go create mode 100644 internal/server/server.go create mode 100644 internal/templates/templates.go create mode 100644 internal/utils/util_test.go create mode 100644 internal/utils/utils.go create mode 100644 main.go create mode 100644 shoelaces-overview.png create mode 100644 test/integ-test/INTEGTEST.md create mode 100644 test/integ-test/expected-results/configs-static-default.txt create mode 100644 test/integ-test/expected-results/ipxemenu.txt create mode 100644 test/integ-test/expected-results/poll-k8s1-1.txt create mode 100644 test/integ-test/expected-results/poll-k8s1-2.txt create mode 100644 test/integ-test/expected-results/poll-k8s1-3-stg.txt create mode 100644 test/integ-test/expected-results/poll-k8s1-4-stg.txt create mode 100644 test/integ-test/expected-results/poll-unknown-set-from-ui.txt create mode 100644 test/integ-test/expected-results/poll-unknown.txt create mode 100644 test/integ-test/expected-results/poll.txt create mode 100644 test/integ-test/expected-results/rc.local-bootstrap create mode 100644 test/integ-test/expected-results/static.html create mode 100644 test/integ-test/integ-test-configs/env_overrides/production/ipxe/coreos.ipxe.slc create mode 100644 test/integ-test/integ-test-configs/env_overrides/staging/preseed/example.preseed.slc create mode 100644 test/integ-test/integ-test-configs/ipxe/coreos.ipxe.slc create mode 100644 test/integ-test/integ-test-configs/mappings.yaml create mode 100644 test/integ-test/integ-test-configs/static/bootstrap.sh create mode 100644 test/integ-test/integ-test-configs/static/rc.local-bootstrap create mode 100755 test/integ-test/integ_test.py create mode 100644 vendor/github.com/go-kit/kit/LICENSE create mode 100644 vendor/github.com/go-kit/kit/log/README.md create mode 100644 vendor/github.com/go-kit/kit/log/doc.go create mode 100644 vendor/github.com/go-kit/kit/log/json_logger.go create mode 100644 vendor/github.com/go-kit/kit/log/level/doc.go create mode 100644 vendor/github.com/go-kit/kit/log/level/level.go create mode 100644 vendor/github.com/go-kit/kit/log/log.go create mode 100644 vendor/github.com/go-kit/kit/log/logfmt_logger.go create mode 100644 vendor/github.com/go-kit/kit/log/nop_logger.go create mode 100644 vendor/github.com/go-kit/kit/log/stdlib.go create mode 100644 vendor/github.com/go-kit/kit/log/sync.go create mode 100644 vendor/github.com/go-kit/kit/log/term/LICENSE create mode 100644 vendor/github.com/go-kit/kit/log/value.go create mode 100644 vendor/github.com/go-logfmt/logfmt/.gitignore create mode 100644 vendor/github.com/go-logfmt/logfmt/.travis.yml create mode 100644 vendor/github.com/go-logfmt/logfmt/LICENSE create mode 100644 vendor/github.com/go-logfmt/logfmt/README.md create mode 100644 vendor/github.com/go-logfmt/logfmt/decode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/doc.go create mode 100644 vendor/github.com/go-logfmt/logfmt/encode.go create mode 100644 vendor/github.com/go-logfmt/logfmt/fuzz.go create mode 100644 vendor/github.com/go-logfmt/logfmt/jsonstring.go create mode 100644 vendor/github.com/go-stack/stack/.travis.yml create mode 100644 vendor/github.com/go-stack/stack/LICENSE.md create mode 100644 vendor/github.com/go-stack/stack/README.md create mode 100644 vendor/github.com/go-stack/stack/stack.go create mode 100644 vendor/github.com/gorilla/context/.travis.yml create mode 100644 vendor/github.com/gorilla/context/LICENSE create mode 100644 vendor/github.com/gorilla/context/README.md create mode 100644 vendor/github.com/gorilla/context/context.go create mode 100644 vendor/github.com/gorilla/context/doc.go create mode 100644 vendor/github.com/gorilla/mux/.travis.yml create mode 100644 vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md create mode 100644 vendor/github.com/gorilla/mux/LICENSE create mode 100644 vendor/github.com/gorilla/mux/README.md create mode 100644 vendor/github.com/gorilla/mux/context_gorilla.go create mode 100644 vendor/github.com/gorilla/mux/context_native.go create mode 100644 vendor/github.com/gorilla/mux/doc.go create mode 100644 vendor/github.com/gorilla/mux/middleware.go create mode 100644 vendor/github.com/gorilla/mux/mux.go create mode 100644 vendor/github.com/gorilla/mux/regexp.go create mode 100644 vendor/github.com/gorilla/mux/route.go create mode 100644 vendor/github.com/gorilla/mux/test_helpers.go create mode 100644 vendor/github.com/justinas/alice/.travis.yml create mode 100644 vendor/github.com/justinas/alice/LICENSE create mode 100644 vendor/github.com/justinas/alice/README.md create mode 100644 vendor/github.com/justinas/alice/chain.go create mode 100644 vendor/github.com/kr/logfmt/.gitignore create mode 100644 vendor/github.com/kr/logfmt/Readme create mode 100644 vendor/github.com/kr/logfmt/decode.go create mode 100644 vendor/github.com/kr/logfmt/scanner.go create mode 100644 vendor/github.com/kr/logfmt/unquote.go create mode 100644 vendor/github.com/namsral/flag/LICENSE create mode 100644 vendor/github.com/namsral/flag/README.md create mode 100644 vendor/github.com/namsral/flag/flag.go create mode 100644 vendor/gopkg.in/yaml.v2/.travis.yml create mode 100644 vendor/gopkg.in/yaml.v2/LICENSE create mode 100644 vendor/gopkg.in/yaml.v2/LICENSE.libyaml create mode 100644 vendor/gopkg.in/yaml.v2/NOTICE create mode 100644 vendor/gopkg.in/yaml.v2/README.md create mode 100644 vendor/gopkg.in/yaml.v2/apic.go create mode 100644 vendor/gopkg.in/yaml.v2/decode.go create mode 100644 vendor/gopkg.in/yaml.v2/emitterc.go create mode 100644 vendor/gopkg.in/yaml.v2/encode.go create mode 100644 vendor/gopkg.in/yaml.v2/go.mod create mode 100644 vendor/gopkg.in/yaml.v2/parserc.go create mode 100644 vendor/gopkg.in/yaml.v2/readerc.go create mode 100644 vendor/gopkg.in/yaml.v2/resolve.go create mode 100644 vendor/gopkg.in/yaml.v2/scannerc.go create mode 100644 vendor/gopkg.in/yaml.v2/sorter.go create mode 100644 vendor/gopkg.in/yaml.v2/writerc.go create mode 100644 vendor/gopkg.in/yaml.v2/yaml.go create mode 100644 vendor/gopkg.in/yaml.v2/yamlh.go create mode 100644 vendor/gopkg.in/yaml.v2/yamlprivateh.go create mode 100644 web/css/bootstrap.css create mode 100644 web/css/bootstrap.min.css create mode 100644 web/css/default.css create mode 100644 web/css/spinner.css create mode 100644 web/fonts/glyphicons-halflings-regular.eot create mode 100644 web/fonts/glyphicons-halflings-regular.svg create mode 100644 web/fonts/glyphicons-halflings-regular.ttf create mode 100644 web/fonts/glyphicons-halflings-regular.woff create mode 100644 web/fonts/glyphicons-halflings-regular.woff2 create mode 100644 web/img/favicon-32x32.png create mode 100644 web/img/shoelaces-logo.png create mode 100644 web/js/bootstrap.js create mode 100644 web/js/bootstrap.min.js create mode 100644 web/js/jquery.min.js create mode 100644 web/js/local.js create mode 100644 web/templates/html/error.html create mode 100644 web/templates/html/events.html create mode 100644 web/templates/html/footer.html create mode 100644 web/templates/html/header.html create mode 100644 web/templates/html/index.html create mode 100644 web/templates/html/mappings.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..74046b8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so +*.pyc + +# Folders +_obj +_test +.vscode +.cache +__pycache__ + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +shoelaces +config.yaml +*.deb diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..3f2c757 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,66 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level" + ] + revision = "ca4112baa34cb55091301bdc13b1420a122b1b9e" + version = "v0.7.0" + +[[projects]] + name = "github.com/go-logfmt/logfmt" + packages = ["."] + revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" + version = "v0.3.0" + +[[projects]] + name = "github.com/go-stack/stack" + packages = ["."] + revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" + version = "v1.7.0" + +[[projects]] + name = "github.com/gorilla/context" + packages = ["."] + revision = "1ea25387ff6f684839d82767c1733ff4d4d15d0a" + version = "v1.1" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "53c1911da2b537f792e7cafcb446b05ffe33b996" + version = "v1.6.1" + +[[projects]] + branch = "master" + name = "github.com/justinas/alice" + packages = ["."] + revision = "03f45bd4b7dad4734bc4620e46a35789349abb20" + +[[projects]] + branch = "master" + name = "github.com/kr/logfmt" + packages = ["."] + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + name = "github.com/namsral/flag" + packages = ["."] + revision = "71ceffbeb0ba60fccc853971bb3ed4d7d90bfd04" + version = "v1.7.4-pre" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "4dbb51d63f4995423447d6189c3a2b9bd4b8d11647b6b1e16a9dbcfd5b66639d" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..bf1b484 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,50 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/go-kit/kit" + version = "0.7.0" + +[[constraint]] + name = "github.com/gorilla/mux" + version = "1.6.1" + +[[constraint]] + branch = "master" + name = "github.com/justinas/alice" + +[[constraint]] + name = "github.com/namsral/flag" + version = "1.7.4-pre" + +[[constraint]] + name = "gopkg.in/yaml.v2" + version = "2.2.1" + +[prune] + go-tests = true + unused-packages = true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7ce9d40 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +GO = go +pkgs = $(shell $(GO) list ./... | grep -v /vendor/) + +all: + $(GO) build + +fmt: + $(GO) fmt + +clean: + rm -f shoelaces + +test: fmt + $(GO) test -v $(pkgs) && \ + ./test/integ-test/integ_test.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..7c71810 --- /dev/null +++ b/README.md @@ -0,0 +1,199 @@ +# **Shoelaces:** lightweight and painless server bootstrapping + +**Shoelaces** serves [iPXE](https://ipxe.org/) boot scripts, +[cloud-init](http://cloud-init.org/) configuration, and any other configuration +files over HTTP to hardware or virtual machines booting over iPXE. It also does +a few other things to make it easier to manage your server deployments: + +* Has a simple but **nice UI** to show the current configuration, and history of + servers that booted. +* Uses simple **Go based template language** to handle more complex configurations. +* Allows specifying the **boot entry point** for a given server based on its + **IP** address or **DNS PTR** record. +* Supports the notion of **environments** for _Development_ and _Production_ + environment configurations, while trying to minimize template duplication. +* Puts unknown servers into iPXE script boot **retry loop**, while at the same + time **showing them in the UI** allowing the user to select a specific boot + configuration. + +## How it works + +![Shoelaces overview](shoelaces-overview.png) + +The previous figure shows a high level overview of how a server contacts +Shoelaces. We can see that as soon as the server boots using network boot, we +instruct the machine to switch to an [iPXE](https://ipxe.org/) ROM. We do this +because we need to be able to make HTTP requests to Shoelaces, and regular +[PXE](https://en.wikipedia.org/wiki/Preboot_Execution_Environment) does not +support that protocol. + +So, when a server boots, the +[DHCP](https://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol) server +will instruct it to retrieve an iPXE ROM from a +[TFTP](https://en.wikipedia.org/wiki/Trivial_File_Transfer_Protocol) server. +When the host receives the iPXE ROM, it will chainload into it and trigger a new +DHCP request. Finally, the server will detect that the request comes from an +iPXE ROM, allowing it to respond with an HTTP URL. This URL, as you may have +guessed, will be pointing to Shoelaces. + + +## Setting up + +### Building Shoelaces +At the moment a binary package is not provided. The only way of running +Shoelaces is to compile it from source. Refer to the Go Programming +Language [Getting Started](https://golang.org/doc/install) guide to learn +how to compile Shoelaces. + +Once that you have configured your Go, you can get and compile Shoelaces by +running: + + $ go get github.com/thousandeyes/shoelaces + $ cd $GOPATH/src/github.com/thousandeyes/shoelaces + $ go build + +### Running Shoelaces +You can quickly try Shoelaces after compiling it by using the example configuration file: + + ./shoelaces -config configs/shoelaces.conf + +Head to [localhost:8081](http://localhost:8081) to checkout Shoelaces' frontend. + +### Shoelaces configuration file +Shoelaces accepts several parameters: + +* `config`: the path to a configuration file. +* `data-dir`: the path to the root directory with the templates. It's advised to + manage the templates in a VCS, such as a git repository. Refer to the [example + data directory](configs/example-templates-configs/) for more information. +* `debug`: enable debug messages. +* `domain`: the domain Shoelaces is going to be listening on. +* `mappings-file`: the path to the YAML mappings file, relative to the `data-dir` parameter. +* `port`: the port Shoelaces will listen on. +* `template-extension`: the filename extension for the templates. The default is + `.slc`, so you can just stick with that. + +The parameters can be specified in a configuration file, as environment +variables or, of course, as parameters when running the Shoelaces binary. + +Refer to the [example config file](configs/shoelaces.conf) for more information. + +### Extra requirements + +Along with your **Shoelaces** installation, you will need a LAN segment with +working [TFTP](https://en.wikipedia.org/wiki/Trivial_File_Transfer_Protocol) and +[DHCP](https://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol) +servers. Any TFTP server should work. The DHCP server will need to be able to +match the `user-class` of the boot client. In our example the configuration is +for the widely used [ISC DHCP Server](https://www.isc.org/downloads/dhcp/). +Shoelaces will happily coexist with the TFTP and DHCP servers on the same host. +The server you are going to bootstrap needs to be capable of booting over the +network using +[PXE](https://en.wikipedia.org/wiki/Preboot_Execution_Environment). + +#### TFTP + +The TFTP server is only used to chainload the iPXE boot loader, so setting it up +in `read-only` mode is sufficient. The loader we use (`undionly.kpxe`) can be +downloaded from the [ipxe.org](http://ipxe.org/howto/chainloading) website. + +It is also possible to compile your own iPXE ROM in order to customize the +booting of your servers. For example, it's useful to [add your own SSL +certificates](http://ipxe.org/crypto#trusted_root_certificates) in case you want +to boot using HTTPS. + +#### DHCP + +Drop this config in your **ISC DHCP** server, replacing the relevant sections +with your TFTP and Shoelaces server addresses. + +``` +# dhcp.conf +next-server ; +if exists user-class and option user-class = "iPXE" { + filename "http:///poll/1/${netX/mac:hexhyp}"; +} else { + filename "undionly.kpxe"; +} +``` + +The **${netX/mac:hexhyp}** strings represents the MAC address of the booting +host. iPXE will be in charge of replacing that string for the actual value. + +*Note*: In case you are using a DHCP server that does not have this level of +flexibility for configuring it, you can always re-compile the iPXE ROM for +[breaking the loop](https://ipxe.org/howto/chainloading#breaking_the_loop_with_an_embedded_script). + + +## Script discoverability + +The purpose of Shoelaces is automation. The less input it receives from the +user, the better. When a server boots, Shoelaces needs the user to select the +booting script to use, but there are certain cases where we can automate even +that. + +* You can preload Shoelaces with mappings from **IPs to boot scripts**. +* You can preload Shoelaces with mappings from **hostnames to boot scripts**. When a + server boots, Shoelaces will make a reverse DNS query to get the hostname for + the IP that made the request, and will match the result to a series of regular + expressions. + +Shoelaces will read these mappings from a YAML file that can be passed as a +program parameter. Refer to the [example mappings +file](configs/example-templates-configs/mappings.yaml) for more information. + +## Environments + +Shoelaces supports the notion of environments a.k.a. *env overrides*. +Consider the following `data-dir` directory structure: +``` +├── cloud-config +│   └── coreos-cloud-config.yaml.slc +├── env_overrides +| └── testing +| └─── cloud-config +| └── coreos-cloud-config.yaml.slc +├── ipxe +│   ├── coreos.ipxe.slc +│   └── ubuntu-minimal.ipxe.slc +├── mappings.yaml +├── preseed +│   └── common.preseed.slc +└── static + ├── bootstrap.sh + └── rc.local-bootstrap +``` + +In this case, hosts that have `environment: testing` set in the `mappings.yaml` +will be assigned the `testing` environment and they'll use the +`coreos-cloud-config.yaml.slc` template from the `env_overrides/testing +directory`, while the rest of the templates will be served from the base +directory. Everything except `mappings.yaml` can be put in `env_overrides/$env` +preserving the path. + +The way this works, considering that **Shoelaces** is mostly stateless, is by +setting different `baseURL` depending on the environment set. Normal requests +would get `baseURL` set to `http://$shoelaces_host:$port` while an environment +request will have `http://$shoelaces_host:$port/env/$environment_name/` + + +*CORNER CASES*: It is not possible to boot a host in a non default environment +unless there is a main iPXE script in the respective override directory. This +means /ipxemenu will only present default and non-default **iPXE** entry points, +and if you have a template that's included later in the boot process as an +override you won't be able to select it. + +## Contributing + +Contributions to Shoelaces are very welcome! Take into account the following +guidelines: + +* [File an issue](https://github.com/thousandeyes/shoelaces/issues) if you find + a bug or, even better, contribute with a pull request. +* We have a bunch of integration tests that can be run by executing `make test`. + Ensure that all test pass before submitting your pull request. +* We are using [dep](https://golang.github.io/dep/) for managing the + dependencies of this project. If your contribution adds a new dependency (not + recommended, but sometimes there isn't other way), please update dep files and + vendor directory + [accordingly](https://golang.github.io/dep/docs/daily-dep.html#adding-a-new-dependency). diff --git a/configs/example-templates-configs/cloud-config/cloud-config-release.slc b/configs/example-templates-configs/cloud-config/cloud-config-release.slc new file mode 100644 index 0000000..0f3de35 --- /dev/null +++ b/configs/example-templates-configs/cloud-config/cloud-config-release.slc @@ -0,0 +1,13 @@ +{{define "cloudconfig-coreos" -}} +#cloud-config +{{template "cloudconfig-coreos-users"}} + +hostname:{{.hostname}} +coreos: + update: + reboot-strategy: off + units: + - name: update-engine.service + enable: true + command: start +{{end}} diff --git a/configs/example-templates-configs/cloud-config/users.slc b/configs/example-templates-configs/cloud-config/users.slc new file mode 100644 index 0000000..6ebdd3f --- /dev/null +++ b/configs/example-templates-configs/cloud-config/users.slc @@ -0,0 +1,11 @@ +{{define "cloudconfig-coreos-users"}} +users: + - name: "shoelaces" + groups: + - "sudo" + - "docker" + ssh-authorized-keys: + - "ssh-rsa fake-key shoelaces@example.com" +ssh_authorized_keys: + - "ssh-rsa fake-key shoelaces@example.com" +{{end}} diff --git a/configs/example-templates-configs/ipxe/coreos.ipxe.slc b/configs/example-templates-configs/ipxe/coreos.ipxe.slc new file mode 100644 index 0000000..64685ff --- /dev/null +++ b/configs/example-templates-configs/ipxe/coreos.ipxe.slc @@ -0,0 +1,16 @@ +{{define "coreos.ipxe" -}} +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/current + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C {{.release}} +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://{{.baseURL}}/configs/cloudconfig-coreos?release={{.release}}&hostname={{.hostname}} console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot +{{end}} \ No newline at end of file diff --git a/configs/example-templates-configs/ipxe/debian.ipxe.slc b/configs/example-templates-configs/ipxe/debian.ipxe.slc new file mode 100644 index 0000000..817b9ff --- /dev/null +++ b/configs/example-templates-configs/ipxe/debian.ipxe.slc @@ -0,0 +1,16 @@ +{{define "debian.ipxe" -}} +#!ipxe + +echo This automatically overwrites data! +echo Debian {{.release}} + +set mirror http://ftp.debian.org/debian/dists/{{.release}}/main/installer-amd64/current/images/netboot/debian-installer/amd64 + +chain http://{{.baseURL}}/configs/linux.cfg?hostname={{.hostname}} + +imgfree + +kernel ${mirror}/linux auto=true priority=critical initrd=initrd.gz preseed/url=http://{{.baseURL}}/configs/debian.preseed ${linuxargs} +initrd ${mirror}/initrd.gz +boot +{{end}} diff --git a/configs/example-templates-configs/ipxe/linux.cfg.slc b/configs/example-templates-configs/ipxe/linux.cfg.slc new file mode 100644 index 0000000..512f7f2 --- /dev/null +++ b/configs/example-templates-configs/ipxe/linux.cfg.slc @@ -0,0 +1,15 @@ +{{define "linux.cfg" -}} +#!ipxe + +# We set VGA to normal and nomodeset because often the framebuffers +# are extraordinarily slow. + +# Two serial interfaces enabled, hopefully one for local and one for remote +# on Dells with iDRAC/iKVM (serial over lan). + +# libata.force=noncq for https://bbs.archlinux.org/viewtopic.php?id=196267 +# We probably want ncq for harddrives, but we don't have many of those. +# queued trim is broken with our 850PRO/Linux combo, or maybe entirely on the Samsung 850 PRO SSDs + +set linuxargs hostname={{.hostname}} console=tty0 console=ttyS0,115200n8 console=ttyS1,115200n8 vga=normal biosdevname=0 nomodeset interface=auto libata.force=noncq consoleblank=0 +{{end}} \ No newline at end of file diff --git a/configs/example-templates-configs/ipxe/storage.ipxe.slc b/configs/example-templates-configs/ipxe/storage.ipxe.slc new file mode 100644 index 0000000..457dad7 --- /dev/null +++ b/configs/example-templates-configs/ipxe/storage.ipxe.slc @@ -0,0 +1,16 @@ +{{define "storage.ipxe" -}} +#!ipxe + +echo This automatically overwrites data! +echo Ubuntu {{.release}} amd64 storage + +set mirror http://mirror.rackspace.com/ubuntu/dists/{{.release}}/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64 + +chain --autofree http://{{.baseURL}}/configs/linux.cfg?hostname={{.hostname}} + +imgfree + +kernel ${mirror}/linux auto=true priority=critical initrd=initrd.gz preseed/url=http://{{.baseURL}}/configs/storage.preseed ${linuxargs} +initrd ${mirror}/initrd.gz +boot +{{end}} \ No newline at end of file diff --git a/configs/example-templates-configs/ipxe/ubuntu-minimal.ipxe.slc b/configs/example-templates-configs/ipxe/ubuntu-minimal.ipxe.slc new file mode 100644 index 0000000..e52f63c --- /dev/null +++ b/configs/example-templates-configs/ipxe/ubuntu-minimal.ipxe.slc @@ -0,0 +1,16 @@ +{{define "ubuntu-minimal.ipxe" -}} +#!ipxe + +echo This automatically overwrites data! +echo Ubuntu {{.release}} minimal + +set mirror http://mirror.rackspace.com/ubuntu/dists/{{.release}}/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64 + +chain http://{{.baseURL}}/configs/linux.cfg?hostname={{.hostname}} + +imgfree + +kernel ${mirror}/linux auto=true priority=critical initrd=initrd.gz preseed/url=http://{{.baseURL}}/configs/ubuntu-minimal.preseed ${linuxargs} +initrd ${mirror}/initrd.gz +boot +{{end}} diff --git a/configs/example-templates-configs/mappings.yaml b/configs/example-templates-configs/mappings.yaml new file mode 100644 index 0000000..dfcb8c9 --- /dev/null +++ b/configs/example-templates-configs/mappings.yaml @@ -0,0 +1,17 @@ +networkMaps: + - network: 192.168.0.0/24 + script: + name: ubuntu-minimal.ipxe + params: + release: xenial + - network: 10.0.10.0/24 + script: + name: coreos.ipxe + params: + release: stable +hostnameMaps: + - hostname: msc1.example.com + script: + name: ubuntu-minimal.ipxe + params: + release: trusty diff --git a/configs/example-templates-configs/preseed/common.preseed.slc b/configs/example-templates-configs/preseed/common.preseed.slc new file mode 100644 index 0000000..bcbf98b --- /dev/null +++ b/configs/example-templates-configs/preseed/common.preseed.slc @@ -0,0 +1,12 @@ +{{define "common.preseed"}} +d-i auto-install/enable boolean true +d-i debian-installer/locale string en_US +d-i console-setup/ask_detect boolean false + +d-i user-setup/allow-password-weak boolean true +d-i passwd/root-login boolean true +d-i passwd/make-user boolean false + +d-i passwd/root-password password password +d-i passwd/root-password-again password password +{{end}} \ No newline at end of file diff --git a/configs/example-templates-configs/preseed/debian.preseed.slc b/configs/example-templates-configs/preseed/debian.preseed.slc new file mode 100644 index 0000000..0915fe3 --- /dev/null +++ b/configs/example-templates-configs/preseed/debian.preseed.slc @@ -0,0 +1,36 @@ +{{define "debian.preseed" -}} +{{template "common.preseed"}} + +d-i partman-auto/method string regular +d-i partman-lvm/device_remove_lvm boolean true +d-i partman-md/device_remove_md boolean true +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true + +# atomic: Just one partition +d-i partman-auto/choose_recipe select atomic + +d-i partman-partitioning/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + +d-i user-setup/encrypt-home boolean false + +d-i base-installer/install-recommends boolean true + +d-i clock-setup/utc boolean true +d-i time/zone string America/Los_Angeles + +d-i clock-setup/ntp boolean true + +d-i debian-installer/quiet boolean false +d-i debian-installer/splash boolean false + +d-i pkgsel/update-policy select unattended-upgrades + +d-i grub-installer/timeout string 5 +d-i popularity-contest/participate boolean false + +d-i finish-install/reboot_in_progress note +{{end}} diff --git a/configs/example-templates-configs/preseed/storage.preseed.slc b/configs/example-templates-configs/preseed/storage.preseed.slc new file mode 100644 index 0000000..982b41b --- /dev/null +++ b/configs/example-templates-configs/preseed/storage.preseed.slc @@ -0,0 +1,104 @@ +{{define "storage.preseed" -}} +{{template "common.preseed"}} + +# The method should be set to "raid". +d-i partman-auto/method string raid +# Specify the disks to be partitioned. They will all get the same layout, +# so this will only work if the disks are the same size. +d-i partman-auto/disk string /dev/sda /dev/sdb + +d-i partman-lvm/device_remove_lvm boolean true +d-i partman-lvm/device_remove_lvm_span boolean true +d-i partman-auto/purge_lvm_from_device boolean true +d-i partman-md/device_remove_md boolean true +d-i partman-lvm/confirm boolean true +d-i partman-auto-lvm/new_vg_name string vg0 + +# Next you need to specify the physical partitions that will be used. +d-i partman-auto/expert_recipe string \ + multiraid :: \ + 256 512 1024 raid \ + $lvmignore{ } \ + $primary{ } \ + method{ raid } \ + raidid{ 1 } \ + . \ + 4000 4096 -1 raid \ + $lvmignore{ } \ + method{ raid } \ + raidid{ 2 } \ + . \ + 2000 4096 50000 ext4 \ + $defaultignore{ } \ + $lvmok{ } \ + method{ format } \ + format{ } \ + use_filesystem{ } \ + filesystem{ ext4 } \ + mountpoint{ / } \ + lv_name{ root } \ + options/noatime{ noatime } \ + options/nodiratime{ nodiratime } \ + options/discard{ discard } \ + . \ + 100% 512 100% linux-swap \ + $defaultignore{ } \ + $lvmok{ } \ + method{ swap } \ + format{ } \ + lv_name{ swap } \ + . + + +# Last you need to specify how the previously defined partitions will be +# used in the RAID setup. Remember to use the correct partition numbers +# for logical partitions. RAID levels 0, 1, 5, 6 and 10 are supported; +# devices are separated using "#". +# Parameters are: +# \ +# + +d-i partman-auto-raid/recipe string \ + 1 2 0 ext3 /boot \ + raidid=1 \ + . \ + 1 2 0 lvm - \ + raidid=2 \ + . + + + +d-i mdadm/boot_degraded boolean true +d-i partman-md/confirm boolean true +d-i partman-partitioning/confirm_write_new_label boolean true +d-i partman/choose_partition select Finish partitioning and write changes to disk +d-i partman/confirm boolean true +d-i partman-md/confirm_nooverwrite boolean true +d-i partman/confirm_nooverwrite boolean true + +d-i user-setup/encrypt-home boolean false + +d-i base-installer/install-recommends boolean false + +d-i clock-setup/utc boolean true +d-i time/zone string UTC + +d-i clock-setup/ntp boolean true + +d-i debian-installer/quiet boolean false +d-i debian-installer/splash boolean false + +##This should be set by Puppet instead. +d-i pkgsel/update-policy select none +## + +d-i grub-installer/timeout string 5 +d-i popularity-contest/participate boolean false + + +d-i finish-install/keep-consoles boolean true +d-i finish-install/reboot_in_progress note + +# This just halts, we want a reboot. +#d-i debian-installer/exit/poweroff boolean true +{{end}} diff --git a/configs/example-templates-configs/preseed/ubuntu-minimal.preseed.slc b/configs/example-templates-configs/preseed/ubuntu-minimal.preseed.slc new file mode 100644 index 0000000..2138764 --- /dev/null +++ b/configs/example-templates-configs/preseed/ubuntu-minimal.preseed.slc @@ -0,0 +1,36 @@ +{{define "ubuntu-minimal.preseed" -}} +{{template "common.preseed"}} + +d-i partman-auto/method string regular +d-i partman-lvm/device_remove_lvm boolean true +d-i partman-md/device_remove_md boolean true +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true + +# atomic: Just one partition +d-i partman-auto/choose_recipe select atomic + +d-i partman-partitioning/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + +d-i user-setup/encrypt-home boolean false + +d-i base-installer/install-recommends boolean true + +d-i clock-setup/utc boolean true +d-i time/zone string America/Los_Angeles + +d-i clock-setup/ntp boolean true + +d-i debian-installer/quiet boolean false +d-i debian-installer/splash boolean false + +d-i pkgsel/update-policy select unattended-upgrades + +d-i grub-installer/timeout string 5 +d-i popularity-contest/participate boolean false + +d-i finish-install/reboot_in_progress note +{{end}} diff --git a/configs/example-templates-configs/static/test-script b/configs/example-templates-configs/static/test-script new file mode 100644 index 0000000..8c33db6 --- /dev/null +++ b/configs/example-templates-configs/static/test-script @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "Hello world" diff --git a/configs/shoelaces.conf b/configs/shoelaces.conf new file mode 100644 index 0000000..fd4230f --- /dev/null +++ b/configs/shoelaces.conf @@ -0,0 +1,6 @@ +port=8081 +domain=localhost +data-dir=configs/example-templates-configs/ +template-extension=.slc +mappings-file=mappings.yaml +debug=true diff --git a/internal/environment/environment.go b/internal/environment/environment.go new file mode 100644 index 0000000..eac0430 --- /dev/null +++ b/internal/environment/environment.go @@ -0,0 +1,173 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment + +import ( + "fmt" + "html/template" + "io/ioutil" + "net" + "os" + "path" + "path/filepath" + "regexp" + "sync" + + "github.com/thousandeyes/shoelaces/internal/event" + "github.com/thousandeyes/shoelaces/internal/log" + "github.com/thousandeyes/shoelaces/internal/mappings" + "github.com/thousandeyes/shoelaces/internal/server" + "github.com/thousandeyes/shoelaces/internal/templates" +) + +// Environment struct holds the shoelaces instance global data. +type Environment struct { + ConfigFile string + BaseURL string + HostnameMaps []mappings.HostnameMap + NetworkMaps []mappings.NetworkMap + ServerStates *server.States + EventLog *event.Log + ParamsBlacklist []string + Templates *templates.ShoelacesTemplates // Dynamic slc templates + StaticTemplates *template.Template // Static Templates + Environments []string // Valid config environments + Logger log.Logger + + Port int + Domain string + DataDir string + StaticDir string + EnvDir string + TemplateExtension string + MappingsFile string + Debug bool +} + +// New returns an initialized environment structure +func New() *Environment { + env := defaultEnvironment() + env.setFlags() + env.validateFlags() + + if env.Debug { + env.Logger = log.AllowDebug(env.Logger) + } + + env.BaseURL = fmt.Sprintf("%s:%d", env.Domain, env.Port) + env.Environments = env.initEnvOverrides() + + env.EventLog = &event.Log{} + + env.Logger.Info("component", "environment", "msg", "Override found", "environment", env.Environments) + + mappingsPath := path.Join(env.DataDir, env.MappingsFile) + if err := env.initMappings(mappingsPath); err != nil { + panic(err) + } + + env.initStaticTemplates() + env.Templates.ParseTemplates(env.Logger, env.DataDir, env.EnvDir, env.Environments, env.TemplateExtension) + server.StartStateCleaner(env.Logger, env.ServerStates) + + return env +} + +func defaultEnvironment() *Environment { + env := &Environment{} + env.NetworkMaps = make([]mappings.NetworkMap, 0) + env.HostnameMaps = make([]mappings.HostnameMap, 0) + env.ServerStates = &server.States{sync.RWMutex{}, make(map[string]*server.State)} + env.ParamsBlacklist = []string{"baseURL"} + env.Templates = templates.New() + env.Environments = make([]string, 0) + env.Logger = log.MakeLogger(os.Stdout) + + return env +} + +func (env *Environment) initStaticTemplates() { + staticTemplates := []string{ + path.Join(env.StaticDir, "templates/html/header.html"), + path.Join(env.StaticDir, "templates/html/index.html"), + path.Join(env.StaticDir, "templates/html/events.html"), + path.Join(env.StaticDir, "templates/html/mappings.html"), + path.Join(env.StaticDir, "templates/html/footer.html"), + } + + fmt.Println(env.StaticDir) + + for _, t := range staticTemplates { + if _, err := os.Stat(t); err != nil { + env.Logger.Error("component", "environment", "msg", "Template does not exists!", "environment", t) + os.Exit(1) + } + } + + env.StaticTemplates = template.Must(template.ParseFiles(staticTemplates...)) +} + +func (env *Environment) initEnvOverrides() []string { + var environments = make([]string, 0) + envPath := filepath.Join(env.DataDir, env.EnvDir) + files, err := ioutil.ReadDir(envPath) + if err == nil { + for _, f := range files { + if f.IsDir() { + environments = append(environments, f.Name()) + } + } + } + return environments +} + +func (env *Environment) initMappings(mappingsPath string) error { + configMappings := mappings.ParseYamlMappings(env.Logger, mappingsPath) + + for _, configNetMap := range configMappings.NetworkMaps { + _, ipnet, err := net.ParseCIDR(configNetMap.Network) + if err != nil { + return err + } + + netMap := mappings.NetworkMap{Network: ipnet, Script: initScript(configNetMap.Script)} + env.NetworkMaps = append(env.NetworkMaps, netMap) + } + + for _, configHostMap := range configMappings.HostnameMaps { + regex, err := regexp.Compile(configHostMap.Hostname) + if err != nil { + return err + } + + hostMap := mappings.HostnameMap{Hostname: regex, Script: initScript(configHostMap.Script)} + env.HostnameMaps = append(env.HostnameMaps, hostMap) + } + + return nil +} + +func initScript(configScript mappings.YamlScript) *mappings.Script { + mappingScript := &mappings.Script{ + Name: configScript.Name, + Environment: configScript.Environment, + Params: make(map[string]interface{}), + } + for key := range configScript.Params { + mappingScript.Params[key] = configScript.Params[key] + } + + return mappingScript +} diff --git a/internal/environment/environment_test.go b/internal/environment/environment_test.go new file mode 100644 index 0000000..8ffe88d --- /dev/null +++ b/internal/environment/environment_test.go @@ -0,0 +1,61 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment + +import ( + "testing" + + "github.com/thousandeyes/shoelaces/internal/mappings" +) + +func TestDefaultEnvironment(t *testing.T) { + env := defaultEnvironment() + if env.BaseURL != "" { + t.Error("BaseURL should be empty string if instantiated directly.") + } + if len(env.HostnameMaps) != 0 { + t.Error("Hostname mappings should be empty") + } + if len(env.NetworkMaps) != 0 { + t.Error("Network mappings should be empty") + } + if len(env.ParamsBlacklist) != 1 && + env.ParamsBlacklist[0] != "baseURL" { + t.Error("ParamsBlacklist should have only baseURL") + } +} + +func TestInitScript(t *testing.T) { + params := make(map[string]string) + params["one"] = "one_value" + configScript := mappings.YamlScript{Name: "testscript", Params: params} + mappingScript := initScript(configScript) + if mappingScript.Name != "testscript" { + t.Errorf("Expected: %s\nGot: %s\n", "testscript", mappingScript.Name) + } + val, ok := mappingScript.Params["one"] + if !ok { + t.Error("Missing param") + } else { + v, ok := val.(string) + if !ok { + t.Error("Bad value type") + } else { + if v != "one_value" { + t.Error("Bad value") + } + } + } +} diff --git a/internal/environment/flags.go b/internal/environment/flags.go new file mode 100644 index 0000000..8250690 --- /dev/null +++ b/internal/environment/flags.go @@ -0,0 +1,57 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment + +import ( + "fmt" + "os" + + "github.com/namsral/flag" +) + +func (env *Environment) setFlags() { + flag.StringVar(&env.ConfigFile, "config", "", "My config file") + flag.IntVar(&env.Port, "port", 8080, "The port where I'm going to listen") + flag.StringVar(&env.Domain, "domain", "localhost", "The address where I'm going to listen") + flag.StringVar(&env.DataDir, "data-dir", "", "Directory with mappings, configs, templates, etc.") + flag.StringVar(&env.StaticDir, "static-dir", "web", "A custom web directory with static files") + flag.StringVar(&env.EnvDir, "env-dir", "env_overrides", "Directory with overrides") + flag.StringVar(&env.TemplateExtension, "template-extension", ".slc", "Shoelaces template extension") + flag.StringVar(&env.MappingsFile, "mappings-file", "mappings.yaml", "My mappings YAML file") + flag.BoolVar(&env.Debug, "debug", false, "Debug mode") + + flag.Parse() +} + +func (env *Environment) validateFlags() { + error := false + + if env.DataDir == "" { + fmt.Println("[*] You must specify the data-dir parameter") + error = true + } + + if env.StaticDir == "" { + fmt.Println("[*] You must specify the data-dir parameter") + error = true + } + + if error { + fmt.Println("\nAvailable parameters:") + flag.PrintDefaults() + fmt.Println("\nParameters can be specified as environment variables, arguments or in a config file.") + os.Exit(1) + } +} diff --git a/internal/event/event.go b/internal/event/event.go new file mode 100644 index 0000000..52db432 --- /dev/null +++ b/internal/event/event.go @@ -0,0 +1,100 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "encoding/json" + "time" + + "github.com/thousandeyes/shoelaces/internal/server" +) + +// Type holds the different typs of events +type Type int + +const ( + // HostPoll is the event generated when a host poll Shoelaces for a script + HostPoll Type = 0 + // UserSelection is the event generated when a user selects a script and hits "Boot!" + UserSelection Type = 1 + // HostBoot is the event generated when a host finally boots + HostBoot Type = 2 + // HostTimeout is the event generated when a host polls and after some + // minutes without activity, timeouts. + HostTimeout Type = 3 + + // PtrMatchBoot is triggered when a PTR is matched to an IP + PtrMatchBoot = "DNS Match" + // SubnetMatchBoot is triggered when an IP matches a subnet mapping + SubnetMatchBoot = "Subnet Match" + // ManualBoot is triggered when the user selects manual boot + ManualBoot = "Manual" +) + +// Event holds information related to the interactions of hosts when they boot. +// It's used exclusively in the Shoelaces web frontend. +type Event struct { + Type Type `json:"eventType"` + Date time.Time `json:"date"` + Server server.Server `json:"server"` + BootType string `json:"bootType"` + Script string `json:"script"` + Message string `json:"message"` + Params map[string]interface{} `json:"params"` +} + +// Log holds the events log +type Log struct { + Events map[string][]Event +} + +// New creates a new Event object +func New(eventType Type, srv server.Server, bootType, script string, params map[string]interface{}) Event { + var event Event + + event.Type = eventType + event.Date = time.Now() + event.Server = srv + event.BootType = bootType + event.Script = script + event.Params = params + + event.setMessage() + + return event +} + +func (e *Event) setMessage() { + switch e.Type { + case HostPoll: + e.Message = "Host " + e.Server.Hostname + " polled for a script." + case UserSelection: + e.Message = "A user selected " + e.Script + " for the host " + e.Server.Hostname + "." + case HostBoot: + params, _ := json.Marshal(e.Params) + e.Message = "Host " + e.Server.Hostname + " booted using " + e.BootType + " method with the following parameters: " + string(params) + case HostTimeout: + e.Message = "Host " + e.Server.Hostname + " timed out." + } +} + +// AddEvent adds an Event into the event log +func (el *Log) AddEvent(eventType Type, srv server.Server, bootType string, script string, params map[string]interface{}) { + if el.Events == nil { + el.Events = make(map[string][]Event) + } + + el.Events[srv.Mac] = append(el.Events[srv.Mac], New(eventType, srv, bootType, script, params)) +} diff --git a/internal/event/event_test.go b/internal/event/event_test.go new file mode 100644 index 0000000..a2eb341 --- /dev/null +++ b/internal/event/event_test.go @@ -0,0 +1,72 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package event + +import ( + "encoding/json" + "testing" + "time" + + "github.com/thousandeyes/shoelaces/internal/server" +) + +const expectedEvent = `{"eventType":0,"date":"1970-01-01T00:00:00Z","server":{"Mac":"","IP":"","Hostname":"test_host"},"bootType":"Manual","script":"freebsd.ipxe","message":"","params":{"baseURL":"localhost:8080","cloudconfig":"virtual","hostname":"","version":"12345"}}` + +func TestNew(t *testing.T) { + event := New(HostPoll, server.Server{Mac: "", IP: "", Hostname: "test_host"}, PtrMatchBoot, "msdos.ipxe", map[string]interface{}{"test": "testParam"}) + if event.Type != HostPoll { + t.Errorf("Expected: \"%d\"\nGot: \"%d\"", HostPoll, event.Type) + } + if event.Server.Hostname != "test_host" { + t.Errorf("Expected: \"test_host\"\nGot: \"%s\"", event.Server.Hostname) + } + if event.BootType != PtrMatchBoot { + t.Errorf("Expected: \"%s\"\nGot: \"%s\"", PtrMatchBoot, event.Server.Hostname) + } + if event.Script != "msdos.ipxe" { + t.Errorf("Expected: \"msdos.ipxe\"\nGot: \"%s\"", event.Server.Hostname) + } + if len(event.Params) != 1 { + t.Error("Expected one parameter") + } + if event.Params["test"] != "testParam" { + t.Error("Expected parameter test: testParam") + } + now := time.Now() + if event.Date.After(now) { + t.Errorf("Expected %s to be after %s", event.Date, now) + } +} + +func TestEventMarshalJSON(t *testing.T) { + event := Event{ + Type: HostPoll, + Date: time.Unix(0, 0).UTC(), + Server: server.Server{Mac: "", IP: "", Hostname: "test_host"}, + BootType: ManualBoot, + Script: "freebsd.ipxe", + Message: "", + Params: map[string]interface{}{ + "baseURL": "localhost:8080", + "cloudconfig": "virtual", + "hostname": "", + "version": "12345", + }, + } + marshaled, _ := json.Marshal(event) + if string(marshaled) != expectedEvent { + t.Errorf("Expected %s\nGot: %s\n", expectedEvent, marshaled) + } +} diff --git a/internal/handlers/common.go b/internal/handlers/common.go new file mode 100644 index 0000000..99511e2 --- /dev/null +++ b/internal/handlers/common.go @@ -0,0 +1,75 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "html/template" + "net/http" + + "github.com/thousandeyes/shoelaces/internal/environment" + "github.com/thousandeyes/shoelaces/internal/ipxe" + "github.com/thousandeyes/shoelaces/internal/mappings" +) + +// DefaultTemplateRenderer holds information for rendering a template based +// on its name. It implements the http.Handler interface. +type DefaultTemplateRenderer struct { + templateName string +} + +// RenderDefaultTemplate renders a template by the given name +func RenderDefaultTemplate(name string) *DefaultTemplateRenderer { + return &DefaultTemplateRenderer{templateName: name} +} + +func (t *DefaultTemplateRenderer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + env := envFromRequest(r) + tpl := env.StaticTemplates + // XXX: Probably not ideal as it's doing the directory listing on every request + ipxeScripts := ipxe.ScriptList(env) + tplVars := struct { + BaseURL string + HostnameMaps *[]mappings.HostnameMap + NetworkMaps *[]mappings.NetworkMap + Scripts *[]ipxe.Script + }{ + env.BaseURL, + &env.HostnameMaps, + &env.NetworkMaps, + &ipxeScripts, + } + renderTemplate(w, tpl, "header", tplVars) + renderTemplate(w, tpl, t.templateName, tplVars) + renderTemplate(w, tpl, "footer", tplVars) +} + +func renderTemplate(w http.ResponseWriter, tpl *template.Template, tmpl string, d interface{}) { + err := tpl.ExecuteTemplate(w, tmpl, d) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +func envFromRequest(r *http.Request) *environment.Environment { + return r.Context().Value(ShoelacesEnvCtxID).(*environment.Environment) +} + +func envNameFromRequest(r *http.Request) string { + e := r.Context().Value(ShoelacesEnvNameCtxID) + if e != nil { + return e.(string) + } + return "" +} diff --git a/internal/handlers/events.go b/internal/handlers/events.go new file mode 100644 index 0000000..f794343 --- /dev/null +++ b/internal/handlers/events.go @@ -0,0 +1,36 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "encoding/json" + "net/http" + "os" +) + +// ListEvents returns a JSON list of the logged events. +func ListEvents(w http.ResponseWriter, r *http.Request) { + // Get Environment and convert the EventLog to JSON + env := envFromRequest(r) + eventList, err := json.Marshal(env.EventLog.Events) + if err != nil { + env.Logger.Error("component", "handler", "err", err) + os.Exit(1) + } + + //Write the EventLog and send the HTTP response + w.Header().Set("Content-Type", "application/json") + w.Write(eventList) +} diff --git a/internal/handlers/ipxemenu.go b/internal/handlers/ipxemenu.go new file mode 100644 index 0000000..0ce8a5a --- /dev/null +++ b/internal/handlers/ipxemenu.go @@ -0,0 +1,64 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "bytes" + "fmt" + "net/http" + + "github.com/thousandeyes/shoelaces/internal/ipxe" +) + +const menuHeader = "#!ipxe\n" + + "chain /poll/1/${netX/mac:hexhyp}\n" + + "menu Choose target to boot\n" + +const menuFooter = "\n" + + "choose target\n" + + "echo -n Enter hostname or none:\n" + + "read hostname\n" + + "set baseurl %s\n" + + "# Boot it as intended.\n" + + "chain ${target}\n" + +// IPXEMenu serves the ipxe menu with list of all available scripts +func IPXEMenu(w http.ResponseWriter, r *http.Request) { + env := envFromRequest(r) + + scripts := ipxe.ScriptList(env) + if len(scripts) == 0 { + http.Error(w, "No Scripts Found", http.StatusInternalServerError) + return + } + + var bootItemsBuffer bytes.Buffer + //Creates the top portion of the iPXE menu + bootItemsBuffer.WriteString(menuHeader) + for _, s := range scripts { + //Formats the bootable scripts separated by newlines into a single string + var desc string + if len(s.Env) > 0 { + desc = fmt.Sprintf("%s [%s]", s.Name, s.Env) + } else { + desc = string(s.Name) + } + bootItem := fmt.Sprintf("item %s%s %s\n", s.Path, s.Name, desc) + bootItemsBuffer.WriteString(bootItem) + } + //Creates the bottom portion of the iPXE menu + bootItemsBuffer.WriteString(fmt.Sprintf(menuFooter, env.BaseURL)) + w.Write(bootItemsBuffer.Bytes()) +} diff --git a/internal/handlers/middleware.go b/internal/handlers/middleware.go new file mode 100644 index 0000000..9525efb --- /dev/null +++ b/internal/handlers/middleware.go @@ -0,0 +1,110 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "github.com/justinas/alice" + "net/http" + "regexp" + + "github.com/thousandeyes/shoelaces/internal/environment" +) + +// ShoelacesCtxID Shoelaces Specific Request Context ID. +type ShoelacesCtxID int + +// ShoelacesEnvCtxID is the context id key for the shoelaces.Environment. +const ShoelacesEnvCtxID ShoelacesCtxID = 0 + +// ShoelacesEnvNameCtxID is the context ID key for the chosen environment. +const ShoelacesEnvNameCtxID ShoelacesCtxID = 1 + +var envRe = regexp.MustCompile(`^(:?/env\/([a-zA-Z0-9_-]+))?(\/.*)`) + +// environmentMiddleware Rewrites the URL in case it was an environment +// specific and sets the environment in the context. +func environmentMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var reqEnv string + m := envRe.FindStringSubmatch(r.URL.Path) + if len(m) > 0 && m[2] != "" { + r.URL.Path = m[3] + reqEnv = m[2] + } + ctx := context.WithValue(r.Context(), ShoelacesEnvNameCtxID, reqEnv) + h.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// loggingMiddleware adds an entry to the logger each time the HTTP service +// receives a request. +func loggingMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + logger := envFromRequest(r).Logger + + logger.Info("component", "http", "type", "request", "src", r.RemoteAddr, "method", r.Method, "url", r.URL) + h.ServeHTTP(w, r) + }) +} + +// SecureHeaders adds secure headers to the responses +func secureHeadersMiddleware(h http.Handler) http.Handler { + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Add X-XSS-Protection header + w.Header().Add("X-XSS-Protection", "1; mode=block") + + // Add X-Content-Type-Options header + w.Header().Add("X-Content-Type-Options", "nosniff") + + // Prevent page from being displayed in an iframe + w.Header().Add("X-Frame-Options", "DENY") + + // Prevent page from being displayed in an iframe + w.Header().Add("Content-Security-Policy", "script-src 'self'") + + h.ServeHTTP(w, r) + }) +} + +// disableCacheMiddleware sets a header for disabling HTTP caching +func disableCacheMiddleware(h http.Handler) http.Handler { + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Cache-Control", "no-cache, no-store, must-revalidate") + + h.ServeHTTP(w, r) + }) +} + +// MiddlewareChain receives a Shoelaces environment and returns a chains of +// middlewares to apply to every request. +func MiddlewareChain(env *environment.Environment) alice.Chain { + // contextMiddleware sets the environment key in the request Context. + contextMiddleware := func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := context.WithValue(r.Context(), ShoelacesEnvCtxID, env) + h.ServeHTTP(w, r.WithContext(ctx)) + }) + } + + return alice.New( + secureHeadersMiddleware, + disableCacheMiddleware, + environmentMiddleware, + contextMiddleware, + loggingMiddleware) +} diff --git a/internal/handlers/polling.go b/internal/handlers/polling.go new file mode 100644 index 0000000..12f36e2 --- /dev/null +++ b/internal/handlers/polling.go @@ -0,0 +1,161 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "os" + + "github.com/gorilla/mux" + "github.com/thousandeyes/shoelaces/internal/log" + "github.com/thousandeyes/shoelaces/internal/polling" + "github.com/thousandeyes/shoelaces/internal/server" + "github.com/thousandeyes/shoelaces/internal/utils" +) + +// PollHandler is called by iPXE boot agents. It returns the boot script +// specified on the configuration or, if the host is unknown, it makes it +// retry for a while until the user specifies alternative IPXE boot script. +func PollHandler(w http.ResponseWriter, r *http.Request) { + env := envFromRequest(r) + + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + vars := mux.Vars(r) + // iPXE MAC addresses come with dashes instead of colons + mac := utils.MacDashToColon(vars["mac"]) + host := r.FormValue("host") + + err = validateMACAndIP(env.Logger, mac, ip) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if host == "" { + host = resolveHostname(env.Logger, ip) + } + + server := server.New(mac, ip, host) + script, err := polling.Poll( + env.Logger, env.ServerStates, env.HostnameMaps, env.NetworkMaps, + env.EventLog, env.Templates, env.BaseURL, server) + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Write([]byte(script)) +} + +// ServerListHandler provides a list of the servers that tried to boot +// but did not match the hostname regex or network mappings. +func ServerListHandler(w http.ResponseWriter, r *http.Request) { + env := envFromRequest(r) + + servers, err := json.Marshal(polling.ListServers(env.ServerStates)) + if err != nil { + env.Logger.Error("component", "handler", "err", err) + os.Exit(1) + } + + w.Header().Set("Content-Type", "application/json") + w.Write(servers) +} + +// UpdateTargetHandler is a POST endpoint that receives parameters for +// booting manually. +func UpdateTargetHandler(w http.ResponseWriter, r *http.Request) { + env := envFromRequest(r) + + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + mac, scriptName, environment, params := parsePostForm(r.PostForm) + if mac == "" || scriptName == "" { + http.Error(w, "MAC address and target must not be empty", http.StatusBadRequest) + return + } + + server := server.New(mac, ip, "") + inputErr, err := polling.UpdateTarget( + env.Logger, env.ServerStates, env.Templates, env.EventLog, env.BaseURL, server, + scriptName, environment, params) + + if err != nil { + if inputErr { + http.Error(w, err.Error(), http.StatusBadRequest) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + http.Redirect(w, r, "/", http.StatusFound) +} + +func parsePostForm(form map[string][]string) (mac, scriptName, environment string, params map[string]interface{}) { + params = make(map[string]interface{}) + for k, v := range form { + if k == "mac" { + mac = utils.MacDashToColon(v[0]) + } else if k == "target" { + scriptName = v[0] + } else if k == "environment" { + environment = v[0] + } else { + params[k] = v[0] + } + } + return +} + +func validateMACAndIP(logger log.Logger, mac string, ip string) (err error) { + if !utils.IsValidMAC(mac) { + logger.Error("component", "polling", "msg", "Invalid MAC", "mac", mac) + return fmt.Errorf("%s", "Invalid MAC") + } + + if !utils.IsValidIP(ip) { + logger.Error("component", "polling", "msg", "Invalid IP", "ip", ip) + return fmt.Errorf("%s", "Invalid IP") + } + + logger.Debug("component", "polling", "msg", "MAC and IP validated", "mac", mac, "ip", ip) + + return nil +} + +func resolveHostname(logger log.Logger, ip string) string { + host := utils.ResolveHostname(ip) + if host == "" { + logger.Info("component", "polling", "msg", "Can't resolve IP", "ip", ip) + } + + return host +} diff --git a/internal/handlers/static.go b/internal/handlers/static.go new file mode 100644 index 0000000..b27fa1a --- /dev/null +++ b/internal/handlers/static.go @@ -0,0 +1,127 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "sort" +) + +// StaticConfigFileHandler handles static config files +type StaticConfigFileHandler struct{} + +func (s *StaticConfigFileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + env := envFromRequest(r) + envName := envNameFromRequest(r) + basePath := path.Join(env.DataDir, "static") + if envName == "" { + http.FileServer(http.Dir(basePath)).ServeHTTP(w, r) + return + } + envPath := filepath.Join(env.DataDir, env.EnvDir, envName, "static") + OverlayFileServer(envPath, basePath).ServeHTTP(w, r) +} + +// StaticConfigFileServer returns a StaticConfigFileHandler instance implementing http.Handler +func StaticConfigFileServer() *StaticConfigFileHandler { + return &StaticConfigFileHandler{} +} + +// OverlayFileServerHandler handles request for overlayer directories +type OverlayFileServerHandler struct { + upper string + lower string +} + +// OverlayFileServer serves static content from two overlayed directories +func OverlayFileServer(upper, lower string) *OverlayFileServerHandler { + return &OverlayFileServerHandler{ + upper: upper, + lower: lower, + } +} + +func (o *OverlayFileServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + + fp := filepath.Clean(r.URL.Path) + upper := filepath.Clean(path.Join(o.upper, fp)) + lower := filepath.Clean(path.Join(o.lower, fp)) + + // TODO: try to avoid stat()-ing both if not necessary + infoUpper, errUpper := os.Stat(upper) + infoLower, errLower := os.Stat(lower) + + // If both upper and lower files/dirs do not exist, return 404 + if errUpper != nil && os.IsNotExist(errUpper) && + errLower != nil && os.IsNotExist(errLower) { + http.NotFound(w, r) + return + } + + isDir := false + fileList := make(map[string]os.FileInfo) + + if errUpper == nil && infoUpper.IsDir() { + files, _ := ioutil.ReadDir(upper) + for _, f := range files { + fileList[f.Name()] = f + } + isDir = true + } + if errLower == nil && infoLower.IsDir() { + files, _ := ioutil.ReadDir(lower) + for _, f := range files { + if _, ok := fileList[f.Name()]; !ok { + fileList[f.Name()] = f + } + } + isDir = true + } + + // Generate HTML directory index + if isDir { + fileListIndex := []string{} + for i := range fileList { + fileListIndex = append(fileListIndex, i) + } + sort.Strings(fileListIndex) + w.Write([]byte("
\n"))
+		for _, i := range fileListIndex {
+			f := fileList[i]
+			name := f.Name()
+			if f.IsDir() {
+				name = name + "/"
+			}
+			l := fmt.Sprintf("%s\n", name, name)
+			w.Write([]byte(l))
+		}
+		w.Write([]byte("
\n")) + return + } + + // Serve the file from the upper layer if it exists. + if errUpper == nil { + http.ServeFile(w, r, upper) + // If not serve it from the lower + } else if errLower == nil { + http.ServeFile(w, r, lower) + } + http.NotFound(w, r) +} diff --git a/internal/handlers/templates.go b/internal/handlers/templates.go new file mode 100644 index 0000000..df1bc58 --- /dev/null +++ b/internal/handlers/templates.go @@ -0,0 +1,83 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/gorilla/mux" + "github.com/thousandeyes/shoelaces/internal/utils" +) + +// TemplateHandler is the dynamic configuration provider endpoint. It +// receives a key and maybe an environment. +func TemplateHandler(w http.ResponseWriter, r *http.Request) { + variablesMap := map[string]interface{}{} + configName := mux.Vars(r)["key"] + + if configName == "" { + http.Error(w, "No template name provided", http.StatusNotFound) + return + } + + for key, val := range r.URL.Query() { + variablesMap[key] = val[0] + } + + env := envFromRequest(r) + envName := envNameFromRequest(r) + variablesMap["baseURL"] = utils.BaseURLforEnvName(env.BaseURL, envName) + + configString, err := env.Templates.RenderTemplate(env.Logger, configName, variablesMap, envName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } else { + io.WriteString(w, configString) + } +} + +// GetTemplateParams receives a script name and returns the parameters +// required for completing that template. +func GetTemplateParams(w http.ResponseWriter, r *http.Request) { + var vars []string + env := envFromRequest(r) + + filterBlacklist := func(s string) bool { + return !utils.StringInSlice(s, env.ParamsBlacklist) + } + + script := r.URL.Query().Get("script") + if script == "" { + http.Error(w, "Required script parameter", http.StatusInternalServerError) + return + } + + envName := r.URL.Query().Get("environment") + if envName == "" { + envName = "default" + } + + vars = utils.Filter(env.Templates.ListVariables(script, envName), filterBlacklist) + + marshaled, err := json.Marshal(vars) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(marshaled) +} diff --git a/internal/ipxe/ipxescript.go b/internal/ipxe/ipxescript.go new file mode 100644 index 0000000..7253195 --- /dev/null +++ b/internal/ipxe/ipxescript.go @@ -0,0 +1,87 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipxe + +import ( + "io/ioutil" + "path/filepath" + "strings" + + "github.com/thousandeyes/shoelaces/internal/environment" + "github.com/thousandeyes/shoelaces/internal/log" +) + +// ScriptName keeps the name of a script +type ScriptName string + +// EnvName holds the name of an environment +type EnvName string + +// ScriptPath holds the path of a script +type ScriptPath string + +// Script holds information regarding an IPXE script. +type Script struct { + Name ScriptName + Env EnvName + Path ScriptPath +} + +// ScriptList receives the global environment and return a list of IPXE +// scripts. +func ScriptList(env *environment.Environment) []Script { + ipxeScripts := make([]Script, 0) + // Collect scripts from the main config dir. + ipxeScripts = appendScriptsFromDir(env.Logger, ipxeScripts, env.TemplateExtension, + filepath.Join(env.DataDir, "ipxe"), "", "/configs/") + + // Collect scripts from the config environments if any + if len(env.Environments) > 0 { + for _, e := range env.Environments { + ep := filepath.Join(env.DataDir, env.EnvDir, e, "ipxe") + ipxeScripts = appendScriptsFromDir(env.Logger, ipxeScripts, env.TemplateExtension, ep, + EnvName(e), ScriptPath("/env/"+e+"/configs/")) + } + } + return ipxeScripts +} + +func appendScriptsFromDir(logger log.Logger, scripts []Script, templateExtension string, dir string, e EnvName, p ScriptPath) []Script { + for _, s := range scriptDirList(logger, templateExtension, dir) { + scripts = append(scripts, Script{Name: s, Env: e, Path: p}) + } + return scripts +} + +// scriptDirList returns the names of all available ipxe script templates +func scriptDirList(logger log.Logger, templateExtension string, datadir string) []ScriptName { + files, err := ioutil.ReadDir(datadir) + if err != nil { + logger.Info("component=ipxescript action=dir-list dir=%s err=\"%v\"", datadir, err.Error()) + return nil + } + + ipxeSuffix := ".ipxe" + suffix := ipxeSuffix + templateExtension + var pxeFiles []ScriptName + for _, f := range files { + // Skip over directories and non-template files. + if f.IsDir() || !strings.HasSuffix(f.Name(), suffix) { + continue + } + pxeFiles = append(pxeFiles, ScriptName(strings.TrimSuffix(f.Name(), templateExtension))) + } + return pxeFiles +} diff --git a/internal/log/log.go b/internal/log/log.go new file mode 100644 index 0000000..071a816 --- /dev/null +++ b/internal/log/log.go @@ -0,0 +1,61 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "io" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" +) + +// Logger struct holds a log.Logger plus functions required for logging +// with different levels. They functions are syntactic sugar to avoid +// having to import "github.com/go-kit/kit/log/level" in every package that +// has to cast a log. +type Logger struct { + Raw log.Logger + Info func(...interface{}) error + Debug func(...interface{}) error + Error func(...interface{}) error +} + +const callerLevel int = 6 + +// MakeLogger receives a io.Writer and return a Logger struct. +func MakeLogger(w io.Writer) Logger { + raw := log.NewLogfmtLogger(log.NewSyncWriter(w)) + raw = log.With(raw, "ts", log.DefaultTimestampUTC, "caller", log.Caller(callerLevel)) + filtered := level.NewFilter(raw, level.AllowInfo()) + + return Logger{ + Raw: raw, + Info: level.Info(filtered).Log, + Debug: level.Debug(filtered).Log, + Error: level.Error(filtered).Log, + } +} + +// AllowDebug receives a Logger and enables the debug logging level. +func AllowDebug(l Logger) Logger { + filtered := level.NewFilter(l.Raw, level.AllowDebug()) + + return Logger{ + Raw: l.Raw, + Info: level.Info(filtered).Log, + Debug: level.Debug(filtered).Log, + Error: level.Error(filtered).Log, + } +} diff --git a/internal/mappings/mappings.go b/internal/mappings/mappings.go new file mode 100644 index 0000000..fba7201 --- /dev/null +++ b/internal/mappings/mappings.go @@ -0,0 +1,80 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mappings + +import ( + "net" + "regexp" + "strings" +) + +// Script holds information related to a booting script. +type Script struct { + Name string + Environment string + Params map[string]interface{} +} + +// NetworkMap struct contains an association between a CIDR network and a +// Script. +type NetworkMap struct { + Network *net.IPNet + Script *Script +} + +// HostnameMap struct contains an association between a hostname regular +// expression and a Script. +type HostnameMap struct { + Hostname *regexp.Regexp + Script *Script +} + +// FindScriptForHostname receives a HostnameMap and a string (that can be a +// regular expression), and tries to find a match in that map. If it finds +// a match, it returns the associated script. +func FindScriptForHostname(maps []HostnameMap, hostname string) (script *Script, ok bool) { + for _, m := range maps { + if m.Hostname.MatchString(hostname) { + return m.Script, true + } + } + return nil, false +} + +// FindScriptForNetwork receives a NetworkMap and an IP and tries to see if +// that IP belongs to any of the configured networks. If it finds a match, +// it returns the associated script. +func FindScriptForNetwork(maps []NetworkMap, ip string) (script *Script, ok bool) { + for _, m := range maps { + if m.Network.Contains(net.ParseIP(ip)) { + return m.Script, true + } + } + return nil, false +} + +func (s Script) String() string { + var result = s.Name + " : { " + elems := []string{} + if s.Environment != "" { + elems = append(elems, "environment: "+s.Environment) + } + for key, value := range s.Params { + elems = append(elems, key+": "+value.(string)) + } + result += strings.Join(elems, ", ") + " }" + + return result +} diff --git a/internal/mappings/mappings_test.go b/internal/mappings/mappings_test.go new file mode 100644 index 0000000..ed45114 --- /dev/null +++ b/internal/mappings/mappings_test.go @@ -0,0 +1,100 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mappings + +import ( + "net" + "regexp" + "testing" +) + +var ( + mockScriptParams1 = map[string]interface{}{ + "param11": "param1_value1", + "param21": "param2_value1", + } + mockScriptParams2 = map[string]interface{}{ + "param12": "param1_value2", + "param22": "param2_value2", + } + mockScript1 = Script{Name: "mock_script1", Params: mockScriptParams1} + mockScript2 = Script{Name: "mock_script2", Params: mockScriptParams2} + + mockRegex1, _ = regexp.Compile("mock_host1") + mockRegex2, _ = regexp.Compile("mock_host2") + + mockHostNameMap1 = HostnameMap{ + Hostname: mockRegex1, + Script: &mockScript1, + } + + mockHostNameMap2 = HostnameMap{ + Hostname: mockRegex2, + Script: &mockScript2, + } + + _, mockNetwork1, _ = net.ParseCIDR("10.0.0.0/8") + _, mockNetwork2, _ = net.ParseCIDR("192.168.0.0/16") + + mockNetworkMap1 = NetworkMap{ + Network: mockNetwork1, + Script: &mockScript1, + } + mockNetworkMap2 = NetworkMap{ + Network: mockNetwork2, + Script: &mockScript2, + } +) + +func TestScript(t *testing.T) { + expected1 := "mock_script1 : { param11: param1_value1, param21: param2_value1 }" + expected2 := "mock_script1 : { param21: param2_value1, param11: param1_value1 }" + mockScriptString := mockScript1.String() + if mockScriptString != expected1 && mockScriptString != expected2 { + t.Errorf("Expected: %s or %s\nGot: %s\n", expected1, expected2, mockScriptString) + } +} + +func TestFindScriptForHostname(t *testing.T) { + maps := []HostnameMap{mockHostNameMap1, mockHostNameMap2} + script, success := FindScriptForHostname(maps, "mock_host1") + if !(script.Name == "mock_script1" && success) { + t.Error("Hostname should have matched") + } + script, success = FindScriptForHostname(maps, "mock_host2") + if !(script.Name == "mock_script2" && success) { + t.Error("Hostname should have matched") + } + script, success = FindScriptForHostname(maps, "mock_host_bad") + if !(script == nil && !success) { + t.Error("Hostname should have not matched") + } +} + +func TestScriptForNetwork(t *testing.T) { + maps := []NetworkMap{mockNetworkMap1, mockNetworkMap2} + script, success := FindScriptForNetwork(maps, "10.0.0.1") + if !(script.Name == "mock_script1" && success) { + t.Error("IP should have matched the network map") + } + script, success = FindScriptForNetwork(maps, "192.168.0.1") + if !(script.Name == "mock_script2" && success) { + t.Error("IP should have matched the network map") + } + script, success = FindScriptForNetwork(maps, "8.8.8.8") + if !(script == nil && !success) { + t.Error("IP shouildn't have matched the network map") + } +} diff --git a/internal/mappings/parse.go b/internal/mappings/parse.go new file mode 100644 index 0000000..64de5bb --- /dev/null +++ b/internal/mappings/parse.go @@ -0,0 +1,78 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mappings + +import ( + "io/ioutil" + "os" + + "gopkg.in/yaml.v2" + + "github.com/thousandeyes/shoelaces/internal/log" +) + +// Mappings struct contains YamlNetworkMaps and YamlHostnameMaps. +type Mappings struct { + NetworkMaps []YamlNetworkMap `yaml:"networkMaps"` + HostnameMaps []YamlHostnameMap `yaml:"hostnameMaps"` +} + +// YamlNetworkMap struct contains an association between a CIDR network and a +// Script. It's different than mapping.NetworkMap in the sense that this +// struct can be used to parse the JSON mapping file. +type YamlNetworkMap struct { + Network string + Script YamlScript +} + +// YamlHostnameMap struct contains an association between a hostname regular +// expression and a Script. It's different than mapping.HostnameMap in the +// sense that this struct can be used to parse the JSON mapping file. +type YamlHostnameMap struct { + Hostname string + Script YamlScript +} + +// YamlScript holds information regarding a script. Its name, its environment +// and its parameters. +type YamlScript struct { + Name string + Environment string + Params map[string]string +} + +// ParseYamlMappings parses the mappings yaml file into a Mappings struct. +func ParseYamlMappings(logger log.Logger, mappingsFile string) *Mappings { + var mappings Mappings + + logger.Info("component", "config", "msg", "Reading mappings", "source", mappingsFile) + yamlFile, err := ioutil.ReadFile(mappingsFile) + + if err != nil { + logger.Error(err) + os.Exit(1) + } + + mappings.NetworkMaps = make([]YamlNetworkMap, 0) + mappings.HostnameMaps = make([]YamlHostnameMap, 0) + + err = yaml.Unmarshal(yamlFile, &mappings) + if err != nil { + logger.Error(err) + os.Exit(1) + } + + return &mappings +} diff --git a/internal/polling/polling.go b/internal/polling/polling.go new file mode 100644 index 0000000..853a751 --- /dev/null +++ b/internal/polling/polling.go @@ -0,0 +1,257 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package polling + +import ( + "bytes" + "errors" + "fmt" + "sort" + "text/template" + "time" + + "github.com/thousandeyes/shoelaces/internal/event" + "github.com/thousandeyes/shoelaces/internal/log" + "github.com/thousandeyes/shoelaces/internal/mappings" + "github.com/thousandeyes/shoelaces/internal/server" + "github.com/thousandeyes/shoelaces/internal/templates" + "github.com/thousandeyes/shoelaces/internal/utils" +) + +// ManualAction represent an action taken when no automatic boot is available. +type ManualAction int + +const ( + maxRetry = 10 + + retryScript = "#!ipxe\n" + + "prompt --key 0x02 --timeout 10000 shoelaces: Press Ctrl-B for manual override... && " + + "chain -ar http://{{.baseURL}}/ipxemenu || " + + "chain -ar http://{{.baseURL}}/poll/1/{{.macAddress}}\n" + + timeoutScript = "#!ipxe\n" + + "exit\n" + + // BootAction is used when a user selects a script for the polling + // server. The server polls once again, so it gets the selected script + // as answer. + BootAction ManualAction = 0 + // RetryAction is used when a server polling does not yet have a script + // selected by the user, hence it has to retry. + RetryAction ManualAction = 1 + // TimeoutAction is used when a server polling is timing out. + TimeoutAction ManualAction = 2 +) + +// ListServers provides a list of the servers that tried to boot +// but did not match the hostname regex or network mappings. +func ListServers(serverStates *server.States) server.Servers { + ret := make([]server.Server, 0) + + serverStates.RLock() + for _, s := range serverStates.Servers { + if s.Target == server.InitTarget { + ret = append(ret, s.Server) + } + } + defer serverStates.RUnlock() + sort.Sort(server.Servers(ret)) + + return ret +} + +// UpdateTarget receives parameters for booting manually. When a host +// didn't match any of the automatic methods for booting, it's going to be +// put on hold. This method is called when something is finally chosen for +// that host. +func UpdateTarget(logger log.Logger, serverStates *server.States, + templateRenderer *templates.ShoelacesTemplates, eventLog *event.Log, baseURL string, srv server.Server, + scriptName string, envName string, params map[string]interface{}) (inputErr bool, err error) { + + if !utils.IsValidMAC(srv.Mac) { + return true, errors.New("Invalid MAC") + } + // Test the template with user inputs + setHostName(params, srv.Mac) + + params["baseURL"] = utils.BaseURLforEnvName(baseURL, envName) + _, err = templateRenderer.RenderTemplate(logger, scriptName, params, envName) + if err != nil { + inputErr = true + return + } + + serverStates.Lock() + defer serverStates.Unlock() + servers := serverStates.Servers + if servers[srv.Mac] == nil { + return true, errors.New("MAC is not in the booting state") + } + + hostname := servers[srv.Mac].Server.Hostname + logger.Debug("component", "polling", "msg", "Setting server override", "server", srv.Mac, "target", scriptName, "environment", envName, "hostname", hostname, "params", params) + eventLog.AddEvent(event.UserSelection, srv, "", scriptName, nil) + servers[srv.Mac].Target = scriptName + servers[srv.Mac].Environment = envName + servers[srv.Mac].Params = params + return false, nil +} + +// Poll contains the main logic of Shoelaces. It uses several heuristics to find +// the right script to return, as network maps, hostname maps and manual +// selection. +func Poll(logger log.Logger, serverStates *server.States, + hostnameMaps []mappings.HostnameMap, networkMaps []mappings.NetworkMap, + eventLog *event.Log, templateRenderer *templates.ShoelacesTemplates, + baseURL string, srv server.Server) (scriptText string, err error) { + + script, found := attemptAutomaticBoot(logger, hostnameMaps, networkMaps, templateRenderer, eventLog, baseURL, srv) + if found { + return script, nil + } + + return manualAction(logger, serverStates, templateRenderer, eventLog, baseURL, srv) +} + +func attemptAutomaticBoot(logger log.Logger, hostnameMaps []mappings.HostnameMap, networkMaps []mappings.NetworkMap, + templateRenderer *templates.ShoelacesTemplates, eventLog *event.Log, + baseURL string, srv server.Server) (scriptText string, found bool) { + + // Find with reverse hostname matched with the hostname regexps + if script, found := mappings.FindScriptForHostname(hostnameMaps, srv.Hostname); found { + logger.Debug("component", "polling", "msg", "Host found", "where", "hostname-mapping", "host", srv.Hostname) + eventLog.AddEvent(event.HostBoot, srv, event.PtrMatchBoot, script.Name, script.Params) + script.Params["hostname"] = srv.Hostname + + return genBootScript(logger, templateRenderer, baseURL, script), found + } + logger.Debug("component", "polling", "msg", "Host not found", "where", "hostname-mapping", "host", srv.Hostname) + + // Find with IP belonging to a configured subnet + if script, found := mappings.FindScriptForNetwork(networkMaps, srv.IP); found { + logger.Debug("component", "polling", "msg", "Host found", "where", "network-mapping", "ip", srv.IP) + setHostName(script.Params, srv.Mac) + srv.Hostname = script.Params["hostname"].(string) + eventLog.AddEvent(event.HostBoot, srv, event.SubnetMatchBoot, script.Name, script.Params) + + return genBootScript(logger, templateRenderer, baseURL, script), found + } + logger.Debug("component", "polling", "msg", "Host not found", "where", "network-mapping", "ip", srv.IP) + + return "", false +} + +func manualAction(logger log.Logger, serverStates *server.States, templateRenderer *templates.ShoelacesTemplates, + eventLog *event.Log, baseURL string, srv server.Server) (scriptText string, err error) { + + script, action := chooseManualAction(logger, serverStates, eventLog, srv) + logger.Debug("component", "polling", "target-script-name", script, "action", action) + + switch action { + case BootAction: + setHostName(script.Params, srv.Mac) + srv.Hostname = script.Params["hostname"].(string) + eventLog.AddEvent(event.HostBoot, srv, event.ManualBoot, script.Name, script.Params) + return genBootScript(logger, templateRenderer, baseURL, script), nil + + case RetryAction: + return genRetryScript(logger, baseURL, srv.Mac), nil + + case TimeoutAction: + return timeoutScript, nil + + default: + logger.Info("component", "polling", "msg", "Unknown action") + return "", fmt.Errorf("%s", "Unknown action") + } +} + +func chooseManualAction(logger log.Logger, serverStates *server.States, + eventLog *event.Log, srv server.Server) (*mappings.Script, ManualAction) { + + serverStates.Lock() + defer serverStates.Unlock() + + if m := serverStates.Servers[srv.Mac]; m != nil { + if m.Target != server.InitTarget { + serverStates.DeleteServer(srv.Mac) + logger.Debug("component", "polling", "msg", "Server boot", "mac", srv.Mac) + return &mappings.Script{ + Name: m.Target, + Environment: m.Environment, + Params: m.Params}, BootAction + } else if m.Retry <= maxRetry { + m.Retry++ + m.LastAccess = int(time.Now().UTC().Unix()) + logger.Debug("component", "polling", "msg", "Retrying reboot", "mac", srv.Mac) + return nil, RetryAction + } else { + serverStates.DeleteServer(srv.Mac) + logger.Debug("component", "polling", "msg", "Timing out server", "mac", srv.Mac) + return nil, TimeoutAction + } + } + + serverStates.AddServer(srv) + logger.Debug("component", "polling", "msg", "New server", "mac", srv.Mac) + eventLog.AddEvent(event.HostPoll, srv, "", "", nil) + + return nil, RetryAction +} + +func setHostName(params map[string]interface{}, mac string) { + if _, ok := params["hostname"]; !ok { + hostname := utils.MacColonToDash(mac) + if hnPrefix, ok := params["hostnamePrefix"]; ok { + hnPrefixStr, isString := hnPrefix.(string) + if !isString { + hnPrefixStr = "" + } + params["hostname"] = hnPrefixStr + hostname + } else { + params["hostname"] = hostname + } + } +} + +func genBootScript(logger log.Logger, templateRenderer *templates.ShoelacesTemplates, baseURL string, script *mappings.Script) string { + script.Params["baseURL"] = utils.BaseURLforEnvName(baseURL, script.Environment) + text, err := templateRenderer.RenderTemplate(logger, script.Name, script.Params, script.Environment) + if err != nil { + panic(err) + } + return text +} + +func genRetryScript(logger log.Logger, baseURL string, mac string) string { + variablesMap := map[string]interface{}{} + parsedTemplate := &bytes.Buffer{} + + tmpl, err := template.New("retry").Parse(retryScript) + if err != nil { + logger.Info("component", "polling", "msg", "Error parsing retry template", "mac", mac) + panic(err) + } + + variablesMap["baseURL"] = baseURL + variablesMap["macAddress"] = utils.MacColonToDash(mac) + err = tmpl.Execute(parsedTemplate, variablesMap) + if err != nil { + logger.Info("component", "polling", "msg", "Error executing retry template", "mac", mac) + panic(err) + } + + return parsedTemplate.String() +} diff --git a/internal/router/router.go b/internal/router/router.go new file mode 100644 index 0000000..abe4f8b --- /dev/null +++ b/internal/router/router.go @@ -0,0 +1,63 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/thousandeyes/shoelaces/internal/environment" + "github.com/thousandeyes/shoelaces/internal/handlers" +) + +// ShoelacesRouter sets up all routes and handlers for shoelaces +func ShoelacesRouter(env *environment.Environment) http.Handler { + r := mux.NewRouter() + + // Main UI page + r.Handle("/", handlers.RenderDefaultTemplate("index")).Methods("GET") + // Event Log History page + r.Handle("/events", handlers.RenderDefaultTemplate("events")).Methods("GET") + // Currently configured mappings page + r.Handle("/mappings", handlers.RenderDefaultTemplate("mappings")).Methods("GET") + // Static files used by the UI + r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", + http.FileServer(http.Dir(env.StaticDir)))) + // Manual boot parameters POST endpoint + r.HandleFunc("/update/target", handlers.UpdateTargetHandler).Methods("POST") + // Provides a list of the servers that tried to boot but did not match + // the hostname regex or network mappings + r.HandleFunc("/ajax/servers", handlers.ServerListHandler).Methods("GET") + // Event Log History JSON endpoint + r.HandleFunc("/ajax/events", handlers.ListEvents).Methods("GET") + // Provides the list of possible parameters for a given template + r.HandleFunc("/ajax/script/params", handlers.GetTemplateParams) + + // Static configuration files endpoint + r.PathPrefix("/configs/static/").Handler(http.StripPrefix("/configs/static/", + handlers.StaticConfigFileServer())) + // Dynamic configuration endpoint + r.HandleFunc("/configs/{key}", handlers.TemplateHandler).Methods("GET") + + // Called by iPXE boot agents, returns boot script specified on the configuration + // or if the host is unknown makes it retry for a while until the user specifies + // alternative ipxe boot script + r.HandleFunc("/poll/1/{mac}", handlers.PollHandler).Methods("GET") + // Serves a generated iPXE boot script providing a selection + // of all of the boot scripts available on the filesystem for that environment. + r.HandleFunc("/ipxemenu", handlers.IPXEMenu).Methods("GET") + + return r +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..1bc7e88 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,123 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "sync" + "time" + + "github.com/thousandeyes/shoelaces/internal/log" +) + +const ( + // InitTarget is an initial dummy target assigned to the servers + InitTarget = "NOTARGET" +) + +// Server holds data that uniquely identifies a server +type Server struct { + Mac string + IP string + Hostname string +} + +// Servers is an array of Server +type Servers []Server + +// Len implementation for the sort Interface +func (s Servers) Len() int { + return len(s) +} + +// Swap implementation for the sort interface +func (s Servers) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +// Less implementation for the Sort interface +func (s Servers) Less(i, j int) bool { + return s[i].Mac < s[j].Mac +} + +// State holds information regarding a host that is attempting to boot. +type State struct { + Server + Target string + Environment string + Params map[string]interface{} + Retry int + LastAccess int +} + +// States holds a map between MAC addresses and +// States. It provides a mutex for thread-safety. +type States struct { + sync.RWMutex + Servers map[string]*State +} + +// New returns a Server with is values initialized +func New(mac string, ip string, hostname string) Server { + return Server{ + Mac: mac, + IP: ip, + Hostname: hostname, + } +} + +// AddServer adds a server to the States struct +func (m *States) AddServer(server Server) { + m.Servers[server.Mac] = &State{ + Server: server, + Target: InitTarget, + Retry: 1, + LastAccess: int(time.Now().UTC().Unix()), + } +} + +// DeleteServer deletes a server from the States struct +func (m *States) DeleteServer(mac string) { + delete(m.Servers, mac) +} + +// StartStateCleaner spawns a goroutine that cleans MAC addresses that +// have been inactive in Shoelaces for more than 3 minutes. +func StartStateCleaner(logger log.Logger, serverStates *States) { + const ( + // 3 minutes + expireAfterSec = 3 * 60 + cleanInterval = time.Minute + ) + // Clean up the server states. Expire after 3 minutes + go func() { + for { + time.Sleep(cleanInterval) + + servers := serverStates.Servers + expire := int(time.Now().UTC().Unix()) - expireAfterSec + + logger.Debug("component", "polling", "msg", "Cleaning", "before", time.Unix(int64(expire), 0)) + + serverStates.Lock() + for mac, state := range servers { + if state.LastAccess <= expire { + delete(servers, mac) + logger.Debug("component", "polling", "msg", "Mac cleaned", "mac", mac) + } + } + serverStates.Unlock() + } + }() +} diff --git a/internal/templates/templates.go b/internal/templates/templates.go new file mode 100644 index 0000000..0df384a --- /dev/null +++ b/internal/templates/templates.go @@ -0,0 +1,233 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package templates + +import ( + "bufio" + "bytes" + "errors" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "text/template" + + "github.com/thousandeyes/shoelaces/internal/log" + "github.com/thousandeyes/shoelaces/internal/utils" +) + +const defaultEnvironment = "default" + +var varRegex = regexp.MustCompile(`{{\.(.*?)}}`) +var configNameRegex = regexp.MustCompile(`{{define\s+"(.*?)".*}}`) + +// ShoelacesTemplates holds the core attributes for handling the dyanmic configurations +// in Shoelaces. +type ShoelacesTemplates struct { + envTemplates map[string]shoelacesTemplateEnvironment + dataDir string + envDir string + tplExt string +} + +type shoelacesTemplateEnvironment struct { + templateObj *template.Template + templateVars map[string][]string +} + +type shoelacesTemplateInfo struct { + name string + variables []string +} + +// New creates and initializes a new ShoelacesTemplates instance a returns a pointer to +// it. +func New() *ShoelacesTemplates { + e := make(map[string]shoelacesTemplateEnvironment) + e[defaultEnvironment] = shoelacesTemplateEnvironment{ + templateObj: template.New(""), + templateVars: make(map[string][]string), + } + return &ShoelacesTemplates{envTemplates: e} +} + +func (s *ShoelacesTemplates) parseTemplateInfo(logger log.Logger, path string) shoelacesTemplateInfo { + fh, err := os.Open(path) + if err != nil { + logger.Error("component", "template", "err", err.Error()) + os.Exit(1) + } + + defer fh.Close() + + templateVars := make([]string, 0) + scanner := bufio.NewScanner(fh) + templateName := "" + i := 0 + for scanner.Scan() { + // find variables + result := varRegex.FindAllStringSubmatch(scanner.Text(), -1) + if varRegex.MatchString(scanner.Text()) { + for _, v := range result { + // we only want the actual match, being second in the group + if !utils.StringInSlice(v[1], templateVars) { + templateVars = append(templateVars, v[1]) + } + } + } + // if first line get name of template + if i == 0 { + nameResult := configNameRegex.FindAllStringSubmatch(scanner.Text(), -1) + templateName = nameResult[0][1] + } + i++ + } + + return shoelacesTemplateInfo{name: templateName, variables: templateVars} +} + +func (s *ShoelacesTemplates) checkAddEnvironment(logger log.Logger, environment string) { + if _, ok := s.envTemplates[environment]; !ok { + c, e := s.envTemplates[defaultEnvironment].templateObj.Clone() + if e != nil { + logger.Error("component", "template", "msg", "Template for environment already executed", "environment", environment) + os.Exit(1) + } + s.envTemplates[environment] = shoelacesTemplateEnvironment{ + templateObj: c, + templateVars: make(map[string][]string), + } + } +} + +func (s *ShoelacesTemplates) addTemplate(logger log.Logger, path string, environment string) error { + s.checkAddEnvironment(logger, environment) + i := s.parseTemplateInfo(logger, path) + _, err := s.envTemplates[environment].templateObj.ParseFiles(path) + if err != nil { + return err + } + s.envTemplates[environment].templateVars[i.name] = i.variables + return nil +} + +func (s *ShoelacesTemplates) getEnvFromPath(path string) string { + envPath := filepath.Join(s.dataDir, s.envDir) + if strings.HasPrefix(path, envPath) { + return strings.Split(strings.TrimPrefix(path, envPath), "/")[1] + } + return defaultEnvironment +} + +// ParseTemplates travels the dataDir and loads in an internal structure +// all the templates found. +func (s *ShoelacesTemplates) ParseTemplates(logger log.Logger, dataDir string, envDir string, envs []string, tplExt string) { + s.dataDir = dataDir + s.envDir = envDir + s.tplExt = tplExt + + logger.Debug("component", "template", "msg", "Template parsing started", "dir", dataDir) + + tplScannerDefault := func(p string, info os.FileInfo, err error) error { + if strings.HasPrefix(p, path.Join(dataDir, envDir)) { + return err + } + if strings.HasSuffix(p, tplExt) { + logger.Info("component", "template", "msg", "Parsing file", "file", p) + if err := s.addTemplate(logger, p, defaultEnvironment); err != nil { + logger.Error("component", "template", "err", err.Error()) + os.Exit(1) + } + } + return err + } + + tplScannerOverride := func(p string, info os.FileInfo, err error) error { + if strings.HasSuffix(p, tplExt) { + env := s.getEnvFromPath(p) + logger.Info("component", "template", "msg", "Parsing ovveride", "environment", env, "file", p) + + if err := s.addTemplate(logger, p, env); err != nil { + logger.Error("component", "template", "err", err.Error()) + os.Exit(1) + } + } + return err + } + + if err := filepath.Walk(dataDir, tplScannerDefault); err != nil { + panic(err) + } + logger.Info("component", "template", "msg", "Parsing override files", "dir", path.Join(dataDir, envDir)) + if err := filepath.Walk(path.Join(dataDir, envDir), tplScannerOverride); err != nil { + logger.Info("component", "template", "msg", "No overrides found") + } + logger.Debug("component", "template", "msg", "Parsing ended") +} + +// RenderTemplate receives a name and a map of parameters, among other +// arguments, and returns the rendered template. It's aware of the +// environment, in case of any. +func (s *ShoelacesTemplates) RenderTemplate(logger log.Logger, configName string, paramMap map[string]interface{}, envName string) (string, error) { + if envName == "" { + envName = defaultEnvironment + } + logger.Info("component", "template", "action", "template-request", "template", configName, "env", envName, "parameters", utils.MapToString(paramMap)) + + requiredVariables := s.envTemplates[envName].templateVars[configName] + + var b bytes.Buffer + err := s.envTemplates[envName].templateObj.ExecuteTemplate(&b, configName, paramMap) + // Fall back to default template in case this is non default environment + // XXX: this is temporary and will be simplified to reduce the code duplication + if err != nil && envName != defaultEnvironment { + requiredVariables = s.envTemplates[defaultEnvironment].templateVars[configName] + err = s.envTemplates[defaultEnvironment].templateObj.ExecuteTemplate(&b, configName, paramMap) + } + if err != nil { + logger.Info("component", "template", "action", "render-template", "err", err.Error()) + return "", err + } + r := b.String() + if strings.Contains(r, "") { + missingVariables := "" + for _, requiredVariable := range requiredVariables { + if !utils.KeyInMap(requiredVariable, paramMap) { + if len(missingVariables) > 0 { + missingVariables += ", " + } + missingVariables += requiredVariable + } + } + logger.Info("component", "template", "msg", "Missing variables in request", "variables", missingVariables) + return "", errors.New("Missing variables in request: " + missingVariables) + } + + return r, nil +} + +// ListVariables receives a template name and return the list of variables +// that belong to it. It's mainly used by the web frontend to provide a +// list of dynamic fields to complete before rendering a template. +func (s *ShoelacesTemplates) ListVariables(templateName, envName string) []string { + if e, ok := s.envTemplates[envName]; ok { + if v, ok := e.templateVars[templateName]; ok { + return v + } + } + var empty []string + return empty +} diff --git a/internal/utils/util_test.go b/internal/utils/util_test.go new file mode 100644 index 0000000..f9daee8 --- /dev/null +++ b/internal/utils/util_test.go @@ -0,0 +1,39 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "testing" +) + +func TestMacColonToDash(t *testing.T) { + testNormMac := func(givenMac, expectedMac string) { + if MacColonToDash(givenMac) != expectedMac { + t.Errorf("Expected: %s\nGot: %s", expectedMac, givenMac) + } + } + testNormMac("ff:ff:ff:ff:ff:ff", "ff-ff-ff-ff-ff-ff") + testNormMac("ff-ff-ff-ff-ff-ff", "ff-ff-ff-ff-ff-ff") +} + +func TestMacDashToColon(t *testing.T) { + testNormMac := func(givenMac, expectedMac string) { + if MacDashToColon(givenMac) != expectedMac { + t.Errorf("Expected: %s\nGot: %s", expectedMac, givenMac) + } + } + testNormMac("ff-ff-ff-ff-ff-ff", "ff:ff:ff:ff:ff:ff") + testNormMac("ff.ff.ff.ff.ff.ff", "ff.ff.ff.ff.ff.ff") +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..b7ee256 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,105 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "net" + "path/filepath" + "strings" +) + +// Filter receives a slide of strings and a function that receives a string +// and returns a bool, and returns a slide that has only the strings that +// returned true when they were applied the received function. +func Filter(files []string, fn func(string) bool) []string { + var ret []string + for _, f := range files { + if fn(f) { + ret = append(ret, f) + } + } + + return ret +} + +// StringInSlice receives a string and a slice of strings and returns true if it exists +// there. +func StringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +// KeyInMap checks wheter the received key exists in the received map. +func KeyInMap(key string, mapInput map[string]interface{}) bool { + _, found := mapInput[key] + return found +} + +// MapToString provides a string representation of a map of strings. +func MapToString(mapInput map[string]interface{}) string { + result := "" + for k, v := range mapInput { + if len(result) > 0 { + result += ", " + } + result += fmt.Sprintf("%s:%v", k, v) + } + return result +} + +// BaseURLforEnvName provides an environment-sensitive method for returning +// the BaseURL of the application. +func BaseURLforEnvName(baseURL, environment string) string { + if environment != "" { + return filepath.Join(baseURL, "env", environment) + } + return baseURL +} + +// ResolveHostname receives an IP and returns the resolved PTR. It returns an +// empty string in case the DNS lookup fails. +func ResolveHostname(ip string) (host string) { + hosts, err := net.LookupAddr(ip) + if err != nil { + return "" + } + return hosts[0] +} + +// IsValidIP returns whether or not an IP is well-formed. +func IsValidIP(ip string) bool { + return net.ParseIP(ip) != nil +} + +// IsValidMAC returns whether or not a MAC address is well-formed. +func IsValidMAC(mac string) bool { + _, err := net.ParseMAC(mac) + return err == nil +} + +// MacColonToDash receives a mac address and replace its colons by dashes +func MacColonToDash(mac string) string { + return strings.Replace(mac, ":", "-", -1) +} + +// MacDashToColon receives a mac address and replace its dashes by colons +func MacDashToColon(mac string) string { + return strings.Replace(mac, "-", ":", -1) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..e66f6e4 --- /dev/null +++ b/main.go @@ -0,0 +1,34 @@ +// Copyright 2018 ThousandEyes Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "os" + + "github.com/thousandeyes/shoelaces/internal/environment" + "github.com/thousandeyes/shoelaces/internal/handlers" + "github.com/thousandeyes/shoelaces/internal/router" +) + +func main() { + env := environment.New() + app := handlers.MiddlewareChain(env).Then(router.ShoelacesRouter(env)) + + env.Logger.Info("component", "main", "transport", "http", "addr", env.BaseURL, "msg", "listening") + env.Logger.Error("component", "main", "err", http.ListenAndServe(env.BaseURL, app)) + + os.Exit(1) +} diff --git a/shoelaces-overview.png b/shoelaces-overview.png new file mode 100644 index 0000000..812fa8b Binary files /dev/null and b/shoelaces-overview.png differ diff --git a/test/integ-test/INTEGTEST.md b/test/integ-test/INTEGTEST.md new file mode 100644 index 0000000..e69de29 diff --git a/test/integ-test/expected-results/configs-static-default.txt b/test/integ-test/expected-results/configs-static-default.txt new file mode 100644 index 0000000..0aa8329 --- /dev/null +++ b/test/integ-test/expected-results/configs-static-default.txt @@ -0,0 +1,4 @@ +
+bootstrap.sh
+rc.local-bootstrap
+
diff --git a/test/integ-test/expected-results/ipxemenu.txt b/test/integ-test/expected-results/ipxemenu.txt new file mode 100644 index 0000000..0560fcc --- /dev/null +++ b/test/integ-test/expected-results/ipxemenu.txt @@ -0,0 +1,12 @@ +#!ipxe +chain /poll/1/${netX/mac:hexhyp} +menu Choose target to boot +item /configs/coreos.ipxe coreos.ipxe +item /env/production/configs/coreos.ipxe coreos.ipxe [production] + +choose target +echo -n Enter hostname or none: +read hostname +set baseurl localhost:18888 +# Boot it as intended. +chain ${target} diff --git a/test/integ-test/expected-results/poll-k8s1-1.txt b/test/integ-test/expected-results/poll-k8s1-1.txt new file mode 100644 index 0000000..46043fd --- /dev/null +++ b/test/integ-test/expected-results/poll-k8s1-1.txt @@ -0,0 +1,14 @@ +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/1122.3.0 + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C stable +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot diff --git a/test/integ-test/expected-results/poll-k8s1-2.txt b/test/integ-test/expected-results/poll-k8s1-2.txt new file mode 100644 index 0000000..46043fd --- /dev/null +++ b/test/integ-test/expected-results/poll-k8s1-2.txt @@ -0,0 +1,14 @@ +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/1122.3.0 + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C stable +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot diff --git a/test/integ-test/expected-results/poll-k8s1-3-stg.txt b/test/integ-test/expected-results/poll-k8s1-3-stg.txt new file mode 100644 index 0000000..46043fd --- /dev/null +++ b/test/integ-test/expected-results/poll-k8s1-3-stg.txt @@ -0,0 +1,14 @@ +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/1122.3.0 + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C stable +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot diff --git a/test/integ-test/expected-results/poll-k8s1-4-stg.txt b/test/integ-test/expected-results/poll-k8s1-4-stg.txt new file mode 100644 index 0000000..6195e9e --- /dev/null +++ b/test/integ-test/expected-results/poll-k8s1-4-stg.txt @@ -0,0 +1,14 @@ +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/1298.6.0 + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C stable +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/env/staging/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot diff --git a/test/integ-test/expected-results/poll-unknown-set-from-ui.txt b/test/integ-test/expected-results/poll-unknown-set-from-ui.txt new file mode 100644 index 0000000..08e161e --- /dev/null +++ b/test/integ-test/expected-results/poll-unknown-set-from-ui.txt @@ -0,0 +1,14 @@ +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/666.0 + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C stable +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-virtual.yaml?release=stable console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot diff --git a/test/integ-test/expected-results/poll-unknown.txt b/test/integ-test/expected-results/poll-unknown.txt new file mode 100644 index 0000000..f17022f --- /dev/null +++ b/test/integ-test/expected-results/poll-unknown.txt @@ -0,0 +1,2 @@ +#!ipxe +prompt --key 0x02 --timeout 10000 shoelaces: Press Ctrl-B for manual override... && chain -ar http://localhost:18888/ipxemenu || chain -ar http://localhost:18888/poll/1/06-66-de-ad-be-ef diff --git a/test/integ-test/expected-results/poll.txt b/test/integ-test/expected-results/poll.txt new file mode 100644 index 0000000..dbba541 --- /dev/null +++ b/test/integ-test/expected-results/poll.txt @@ -0,0 +1,2 @@ +#!ipxe +prompt --key 0x02 --timeout 10000 shoelaces: Press Ctrl-B for manual override... && chain -ar http://localhost:18888/ipxemenu || chain -ar http://localhost:18888/poll/1/ff-ff-ff-ff-ff-ff diff --git a/test/integ-test/expected-results/rc.local-bootstrap b/test/integ-test/expected-results/rc.local-bootstrap new file mode 100644 index 0000000..084ccc2 --- /dev/null +++ b/test/integ-test/expected-results/rc.local-bootstrap @@ -0,0 +1,4 @@ +#!/bin/sh + +/usr/local/sbin/bootstrap > /var/log/bootstrap.log 2>&1 & +exit 0 diff --git a/test/integ-test/expected-results/static.html b/test/integ-test/expected-results/static.html new file mode 100644 index 0000000..eff16d6 --- /dev/null +++ b/test/integ-test/expected-results/static.html @@ -0,0 +1,7 @@ +
+css/
+fonts/
+img/
+js/
+templates/
+
diff --git a/test/integ-test/integ-test-configs/env_overrides/production/ipxe/coreos.ipxe.slc b/test/integ-test/integ-test-configs/env_overrides/production/ipxe/coreos.ipxe.slc new file mode 100644 index 0000000..f60b478 --- /dev/null +++ b/test/integ-test/integ-test-configs/env_overrides/production/ipxe/coreos.ipxe.slc @@ -0,0 +1,16 @@ +{{define "coreos.ipxe" -}} +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/{{.version}} + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C stable +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://{{.baseURL}}/configs/coreos-{{.cloudconfig}}.yaml?release=stable&hostname={{.hostname}} console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot +{{end}} diff --git a/test/integ-test/integ-test-configs/env_overrides/staging/preseed/example.preseed.slc b/test/integ-test/integ-test-configs/env_overrides/staging/preseed/example.preseed.slc new file mode 100644 index 0000000..824bcdd --- /dev/null +++ b/test/integ-test/integ-test-configs/env_overrides/staging/preseed/example.preseed.slc @@ -0,0 +1,14 @@ +{{define "example.preseed" -}} + +d-i partman-auto-raid/recipe string \ + 1 4 0 ext3 /boot \ + /dev/sda1#/dev/sdb1#/dev/sdc1#/dev/sdd1 \ + . \ + 10 4 0 lvm - \ + /dev/sda5#/dev/sdb5#/dev/sdc5#/dev/sdd5 \ + . + +d-i partman-auto/disk string /dev/sda /dev/sdb /dev/sdc /dev/sdd +d-i grub-installer/bootdev string /dev/sda /dev/sdb /dev/sdc /dev/sdd + +{{end}} diff --git a/test/integ-test/integ-test-configs/ipxe/coreos.ipxe.slc b/test/integ-test/integ-test-configs/ipxe/coreos.ipxe.slc new file mode 100644 index 0000000..7cb5156 --- /dev/null +++ b/test/integ-test/integ-test-configs/ipxe/coreos.ipxe.slc @@ -0,0 +1,16 @@ +{{define "coreos.ipxe" -}} +#!ipxe + +set coreos-url http://stable.release.core-os.net/amd64-usr/{{.version}} + +echo This will currently autologin into tty1 on the console. +echo From there you can su to root and install CoreOS to disk using: +echo coreos-install -d /dev/sda -C stable +echo You will probably need to chroot into /dev/sda9 to configure accounts. +echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/ + +kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://{{.baseURL}}/configs/coreos-{{.cloudconfig}}.yaml?release=stable console=tty1 coreos.autologin=tty1 +initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz + +boot +{{end}} diff --git a/test/integ-test/integ-test-configs/mappings.yaml b/test/integ-test/integ-test-configs/mappings.yaml new file mode 100644 index 0000000..534b72f --- /dev/null +++ b/test/integ-test/integ-test-configs/mappings.yaml @@ -0,0 +1,39 @@ +networkMaps: + - network: 20.20.20.20/24 + script: + name: ubuntu-minimal.ipxe + params: + hostname: placeholder + +hostnameMaps: + - hostname: '(etcd|k8s)\d-m\d' + script: + name: coreos.ipxe + params: + version: 1122.3.0 + cloudconfig: virtual + - hostname: '(etcd|k8s)\d-m\d' + script: + name: coreos.ipxe + params: + version: 1122.3.0 + cloudconfig: virtual + - hostname: 'k8s1-4' + script: + name: coreos.ipxe + environment: staging + params: + version: 1298.6.0 + cloudconfig: baremetal + - hostname: 'k8s1-\d' + script: + name: coreos.ipxe + params: + version: 1122.3.0 + cloudconfig: baremetal + - hostname: 'k8s1-\d' + script: + name: coreos.ipxe + params: + version: 1122.3.0 + cloudconfig: baremetal diff --git a/test/integ-test/integ-test-configs/static/bootstrap.sh b/test/integ-test/integ-test-configs/static/bootstrap.sh new file mode 100644 index 0000000..847ae99 --- /dev/null +++ b/test/integ-test/integ-test-configs/static/bootstrap.sh @@ -0,0 +1,11 @@ +#!/bin/bash +export DEBIAN_FRONTEND=noninteractive + +echo Example boostrap configuration +apt-get install hello + +echo '#!/bin/sh +exit 0' > /etc/rc.local + +# Don't want to run this accidentally. +chmod 0 /usr/local/sbin/bootstrap diff --git a/test/integ-test/integ-test-configs/static/rc.local-bootstrap b/test/integ-test/integ-test-configs/static/rc.local-bootstrap new file mode 100644 index 0000000..084ccc2 --- /dev/null +++ b/test/integ-test/integ-test-configs/static/rc.local-bootstrap @@ -0,0 +1,4 @@ +#!/bin/sh + +/usr/local/sbin/bootstrap > /var/log/bootstrap.log 2>&1 & +exit 0 diff --git a/test/integ-test/integ_test.py b/test/integ-test/integ_test.py new file mode 100755 index 0000000..4f02fe3 --- /dev/null +++ b/test/integ-test/integ_test.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python + +# Copyright 2018 ThousandEyes Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Test shoelaces """ + +import os +import signal +import subprocess +import sys +import time +import tempfile +import string +import pytest +import requests +import datetime +import dateutil.parser +from requests.exceptions import RequestException + +API_HOST = 'localhost' +API_PORT = '18888' +API_URL = "http://{}:{}".format(API_HOST, API_PORT) +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +BASE_DIR = os.path.dirname(os.path.dirname(TEST_DIR)) +FIXTURE_DIR = os.path.join(TEST_DIR, 'expected-results') +STATIC_DIR = os.path.join(BASE_DIR, "web") +SHOELACES_BINARY = os.path.join(BASE_DIR, "shoelaces") + + +@pytest.fixture(scope="session", autouse=True) +def shoelaces_binary(): + os.chdir(BASE_DIR) + subprocess.check_call(["go", "build"]) + os.chdir(TEST_DIR) + + +@pytest.fixture(scope="session", autouse=True) +def config_file(shoelaces_binary): + """ Create a temporary config file """ + temp_config_tpl = string.Template("domain=$host\n" + "port=$port\n" + "data-dir=integ-test-configs\n" + "static-dir=$static_dir\n" + "template-extension=.slc\n" + "mappings-file=mappings.yaml\n" + "debug=true\n") + temp_config = temp_config_tpl.substitute(host=API_HOST, + port=API_PORT, + static_dir=STATIC_DIR) + + sys.stderr.write("Using:\n{}".format(temp_config)) + temp_cfg_file = tempfile.NamedTemporaryFile(delete=False) + temp_cfg_file.write(temp_config) + temp_cfg_file.flush() + temp_cfg_file_name = temp_cfg_file.name + temp_cfg_file.close() + yield temp_cfg_file_name + os.unlink(temp_cfg_file_name) + + +@pytest.fixture(scope="session", autouse=True) +def shoelaces_instance(config_file): + """ Shoelaces test fixture. """ + shoelaces_start_cmd = [SHOELACES_BINARY, "-config", config_file] + shoelaces = subprocess.Popen(shoelaces_start_cmd, preexec_fn=os.setsid) + sys.stderr.write("\nStarting Shoelaces...\n") + yield shoelaces + sys.stderr.write("\nShutting down Shoelaces...\n") + os.killpg(os.getpgid(shoelaces.pid), signal.SIGTERM) + sys.stderr.write("\nDone\n") + + +def test_shoelaces_startup(shoelaces_instance): + """ Test API liveness """ + attempts = 0 + while True: + try: + req = requests.get('{}/'.format(API_URL)) + req.raise_for_status() + sys.stderr.write('\n\nApi startup successful.\n') + break + except RequestException: + attempts += 1 + if attempts > 10: + raise + sys.stderr.write(".") + time.sleep(1) + + +@pytest.mark.parametrize(("path"), [("/"), ("/events"), ("/mappings")]) +def test_response_success(shoelaces_instance, path): + r = requests.get("{}{}".format(API_URL, path)) + r.raise_for_status() + + +REQUEST_RESPONSE_PAIRS = [("/static/", "static.html"), + ("/configs/static/", "configs-static-default.txt"), + ("/configs/static/rc.local-bootstrap", + "rc.local-bootstrap"), + ("/ipxemenu", "ipxemenu.txt")] + + +@pytest.mark.parametrize(("request_path", "response_file"), REQUEST_RESPONSE_PAIRS) +def test_request_response(shoelaces_instance, request_path, response_file): + with open(os.path.join(FIXTURE_DIR, response_file)) as response_body: + assert requests.get( + API_URL + request_path).text == response_body.read() + + +def gen_mac_server_pairs(): + generated = [] + for m in range(0x00, 0x100, 0x11): + o = "{:02x}".format(m) + generated.append({'IP': '127.0.0.1', 'Mac': "ff:ff:ff:ff:ff:{}".format(o), 'Hostname': 'localhost'}) + yield (o, list(generated)) + + +@pytest.mark.parametrize(("mac_last_octet", "servers"), gen_mac_server_pairs()) +def test_servers(shoelaces_instance, mac_last_octet, servers): + poll_url = "{}/poll/1/ff-ff-ff-ff-ff-{}".format(API_URL, mac_last_octet) + req = requests.get(poll_url) + req = requests.get("{}/ajax/servers".format(API_URL)) + assert sorted(req.json()) == sorted(servers) + + +def test_unknown_server(shoelaces_instance): + poll_url = "{}/poll/1/06-66-de-ad-be-ef".format(API_URL) + # Request for unknown host will give result in retries/polling + with open(os.path.join(FIXTURE_DIR, "poll-unknown.txt")) as poll: + assert requests.get(poll_url).text == poll.read() + # Setting the config for the new host should succeed. + requests.post(API_URL + '/update/target', + {"target": "coreos.ipxe", + "mac": "06:66:de:ad:be:ef", + "version": "666.0", + "cloudconfig": "virtual"}).raise_for_status() + # After setting we should be able to get the new config. + with open(os.path.join(FIXTURE_DIR, "poll-unknown-set-from-ui.txt")) as poll: + assert requests.get(poll_url).text == poll.read() + # Once fetched the host is now again "unknown" + with open(os.path.join(FIXTURE_DIR, "poll-unknown.txt")) as poll: + assert requests.get(poll_url).text == poll.read() + + +def test_events(shoelaces_instance): + url = "{}/ajax/events".format(API_URL) + req = requests.get(url) + req.raise_for_status() + res = req.json() + # assert mac is in dictionary + assert '06:66:de:ad:be:ef' in res + # assert array with one element + assert isinstance(res['06:66:de:ad:be:ef'], list) and len(res['06:66:de:ad:be:ef']) == 4 + # assert we have a date field + assert 'date' in res['06:66:de:ad:be:ef'][0] + # assert our date actually parses + assert dateutil.parser.parse(res['06:66:de:ad:be:ef'][0]['date']) + del res['06:66:de:ad:be:ef'][0]['date'] + # compare to the expected result sans the date as it would be different + assert sorted(res['06:66:de:ad:be:ef'][0]) == sorted({'eventType': '0', + 'message': '0', + 'bootType': 'Manual', + 'server': {'mac':'', + 'ip': '', + 'hostname': '06-66-de-ad-be-ef'}, + 'params': {'baseURL': 'localhost:18888', + 'cloudconfig': 'virtual', + 'hostname': '06-66-de-ad-be-ef', + 'version': '666.0'}, + 'script': 'coreos.ipxe'}) + + +POLL_PAIRS = [(None, "poll.txt"), + ({"host": "k8s1-3"}, "poll-k8s1-3-stg.txt"), + ({"host": "k8s1-4"}, "poll-k8s1-4-stg.txt"), + ({"host": "k8s1-1"}, "poll-k8s1-1.txt"), + ({"host": "k8s1-2"}, "poll-k8s1-2.txt")] + + +@pytest.mark.parametrize(("params", "expected"), POLL_PAIRS) +def test_poll(shoelaces_instance, params, expected): + """ Test Poll handler """ + poll_url = "{}/poll/1/ff-ff-ff-ff-ff-ff".format(API_URL) + req = requests.get(poll_url, params=params) + req.raise_for_status() + with open(os.path.join(FIXTURE_DIR, expected), 'r') as poll: + assert poll.read() == req.text + + +TPL_VARS_PAIRS = [("coreos.ipxe", "", ["cloudconfig", "version"]), + ("coreos.ipxe", "default", ["cloudconfig", "version"]), + ("coreos.ipxe", "production", ["cloudconfig", "version", "hostname"])] + + +@pytest.mark.parametrize(("script", "env", "vars"), TPL_VARS_PAIRS) +def test_template_variables_list(shoelaces_instance, script, env, vars): + url = "{}/ajax/script/params".format(API_URL) + req = requests.get(url, params={"script": script, "environment": env}) + req.raise_for_status() + assert sorted(req.json()) == sorted(vars) + + +if __name__ == "__main__": + pytest.main(args=['-v'], plugins=None) diff --git a/vendor/github.com/go-kit/kit/LICENSE b/vendor/github.com/go-kit/kit/LICENSE new file mode 100644 index 0000000..9d83342 --- /dev/null +++ b/vendor/github.com/go-kit/kit/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Peter Bourgon + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/vendor/github.com/go-kit/kit/log/README.md b/vendor/github.com/go-kit/kit/log/README.md new file mode 100644 index 0000000..7222f80 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/README.md @@ -0,0 +1,147 @@ +# package log + +`package log` provides a minimal interface for structured logging in services. +It may be wrapped to encode conventions, enforce type-safety, provide leveled +logging, and so on. It can be used for both typical application log events, +and log-structured data streams. + +## Structured logging + +Structured logging is, basically, conceding to the reality that logs are +_data_, and warrant some level of schematic rigor. Using a stricter, +key/value-oriented message format for our logs, containing contextual and +semantic information, makes it much easier to get insight into the +operational activity of the systems we build. Consequently, `package log` is +of the strong belief that "[the benefits of structured logging outweigh the +minimal effort involved](https://www.thoughtworks.com/radar/techniques/structured-logging)". + +Migrating from unstructured to structured logging is probably a lot easier +than you'd expect. + +```go +// Unstructured +log.Printf("HTTP server listening on %s", addr) + +// Structured +logger.Log("transport", "HTTP", "addr", addr, "msg", "listening") +``` + +## Usage + +### Typical application logging + +```go +w := log.NewSyncWriter(os.Stderr) +logger := log.NewLogfmtLogger(w) +logger.Log("question", "what is the meaning of life?", "answer", 42) + +// Output: +// question="what is the meaning of life?" answer=42 +``` + +### Contextual Loggers + +```go +func main() { + var logger log.Logger + logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) + logger = log.With(logger, "instance_id", 123) + + logger.Log("msg", "starting") + NewWorker(log.With(logger, "component", "worker")).Run() + NewSlacker(log.With(logger, "component", "slacker")).Run() +} + +// Output: +// instance_id=123 msg=starting +// instance_id=123 component=worker msg=running +// instance_id=123 component=slacker msg=running +``` + +### Interact with stdlib logger + +Redirect stdlib logger to Go kit logger. + +```go +import ( + "os" + stdlog "log" + kitlog "github.com/go-kit/kit/log" +) + +func main() { + logger := kitlog.NewJSONLogger(kitlog.NewSyncWriter(os.Stdout)) + stdlog.SetOutput(kitlog.NewStdlibAdapter(logger)) + stdlog.Print("I sure like pie") +} + +// Output: +// {"msg":"I sure like pie","ts":"2016/01/01 12:34:56"} +``` + +Or, if, for legacy reasons, you need to pipe all of your logging through the +stdlib log package, you can redirect Go kit logger to the stdlib logger. + +```go +logger := kitlog.NewLogfmtLogger(kitlog.StdlibWriter{}) +logger.Log("legacy", true, "msg", "at least it's something") + +// Output: +// 2016/01/01 12:34:56 legacy=true msg="at least it's something" +``` + +### Timestamps and callers + +```go +var logger log.Logger +logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) +logger = log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) + +logger.Log("msg", "hello") + +// Output: +// ts=2016-01-01T12:34:56Z caller=main.go:15 msg=hello +``` + +## Supported output formats + +- [Logfmt](https://brandur.org/logfmt) ([see also](https://blog.codeship.com/logfmt-a-log-format-thats-easy-to-read-and-write)) +- JSON + +## Enhancements + +`package log` is centered on the one-method Logger interface. + +```go +type Logger interface { + Log(keyvals ...interface{}) error +} +``` + +This interface, and its supporting code like is the product of much iteration +and evaluation. For more details on the evolution of the Logger interface, +see [The Hunt for a Logger Interface](http://go-talks.appspot.com/github.com/ChrisHines/talks/structured-logging/structured-logging.slide#1), +a talk by [Chris Hines](https://github.com/ChrisHines). +Also, please see +[#63](https://github.com/go-kit/kit/issues/63), +[#76](https://github.com/go-kit/kit/pull/76), +[#131](https://github.com/go-kit/kit/issues/131), +[#157](https://github.com/go-kit/kit/pull/157), +[#164](https://github.com/go-kit/kit/issues/164), and +[#252](https://github.com/go-kit/kit/pull/252) +to review historical conversations about package log and the Logger interface. + +Value-add packages and suggestions, +like improvements to [the leveled logger](https://godoc.org/github.com/go-kit/kit/log/level), +are of course welcome. Good proposals should + +- Be composable with [contextual loggers](https://godoc.org/github.com/go-kit/kit/log#With), +- Not break the behavior of [log.Caller](https://godoc.org/github.com/go-kit/kit/log#Caller) in any wrapped contextual loggers, and +- Be friendly to packages that accept only an unadorned log.Logger. + +## Benchmarks & comparisons + +There are a few Go logging benchmarks and comparisons that include Go kit's package log. + +- [imkira/go-loggers-bench](https://github.com/imkira/go-loggers-bench) includes kit/log +- [uber-common/zap](https://github.com/uber-common/zap), a zero-alloc logging library, includes a comparison with kit/log diff --git a/vendor/github.com/go-kit/kit/log/doc.go b/vendor/github.com/go-kit/kit/log/doc.go new file mode 100644 index 0000000..918c0af --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/doc.go @@ -0,0 +1,116 @@ +// Package log provides a structured logger. +// +// Structured logging produces logs easily consumed later by humans or +// machines. Humans might be interested in debugging errors, or tracing +// specific requests. Machines might be interested in counting interesting +// events, or aggregating information for off-line processing. In both cases, +// it is important that the log messages are structured and actionable. +// Package log is designed to encourage both of these best practices. +// +// Basic Usage +// +// The fundamental interface is Logger. Loggers create log events from +// key/value data. The Logger interface has a single method, Log, which +// accepts a sequence of alternating key/value pairs, which this package names +// keyvals. +// +// type Logger interface { +// Log(keyvals ...interface{}) error +// } +// +// Here is an example of a function using a Logger to create log events. +// +// func RunTask(task Task, logger log.Logger) string { +// logger.Log("taskID", task.ID, "event", "starting task") +// ... +// logger.Log("taskID", task.ID, "event", "task complete") +// } +// +// The keys in the above example are "taskID" and "event". The values are +// task.ID, "starting task", and "task complete". Every key is followed +// immediately by its value. +// +// Keys are usually plain strings. Values may be any type that has a sensible +// encoding in the chosen log format. With structured logging it is a good +// idea to log simple values without formatting them. This practice allows +// the chosen logger to encode values in the most appropriate way. +// +// Contextual Loggers +// +// A contextual logger stores keyvals that it includes in all log events. +// Building appropriate contextual loggers reduces repetition and aids +// consistency in the resulting log output. With and WithPrefix add context to +// a logger. We can use With to improve the RunTask example. +// +// func RunTask(task Task, logger log.Logger) string { +// logger = log.With(logger, "taskID", task.ID) +// logger.Log("event", "starting task") +// ... +// taskHelper(task.Cmd, logger) +// ... +// logger.Log("event", "task complete") +// } +// +// The improved version emits the same log events as the original for the +// first and last calls to Log. Passing the contextual logger to taskHelper +// enables each log event created by taskHelper to include the task.ID even +// though taskHelper does not have access to that value. Using contextual +// loggers this way simplifies producing log output that enables tracing the +// life cycle of individual tasks. (See the Contextual example for the full +// code of the above snippet.) +// +// Dynamic Contextual Values +// +// A Valuer function stored in a contextual logger generates a new value each +// time an event is logged. The Valuer example demonstrates how this feature +// works. +// +// Valuers provide the basis for consistently logging timestamps and source +// code location. The log package defines several valuers for that purpose. +// See Timestamp, DefaultTimestamp, DefaultTimestampUTC, Caller, and +// DefaultCaller. A common logger initialization sequence that ensures all log +// entries contain a timestamp and source location looks like this: +// +// logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) +// logger = log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) +// +// Concurrent Safety +// +// Applications with multiple goroutines want each log event written to the +// same logger to remain separate from other log events. Package log provides +// two simple solutions for concurrent safe logging. +// +// NewSyncWriter wraps an io.Writer and serializes each call to its Write +// method. Using a SyncWriter has the benefit that the smallest practical +// portion of the logging logic is performed within a mutex, but it requires +// the formatting Logger to make only one call to Write per log event. +// +// NewSyncLogger wraps any Logger and serializes each call to its Log method. +// Using a SyncLogger has the benefit that it guarantees each log event is +// handled atomically within the wrapped logger, but it typically serializes +// both the formatting and output logic. Use a SyncLogger if the formatting +// logger may perform multiple writes per log event. +// +// Error Handling +// +// This package relies on the practice of wrapping or decorating loggers with +// other loggers to provide composable pieces of functionality. It also means +// that Logger.Log must return an error because some +// implementations—especially those that output log data to an io.Writer—may +// encounter errors that cannot be handled locally. This in turn means that +// Loggers that wrap other loggers should return errors from the wrapped +// logger up the stack. +// +// Fortunately, the decorator pattern also provides a way to avoid the +// necessity to check for errors every time an application calls Logger.Log. +// An application required to panic whenever its Logger encounters +// an error could initialize its logger as follows. +// +// fmtlogger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) +// logger := log.LoggerFunc(func(keyvals ...interface{}) error { +// if err := fmtlogger.Log(keyvals...); err != nil { +// panic(err) +// } +// return nil +// }) +package log diff --git a/vendor/github.com/go-kit/kit/log/json_logger.go b/vendor/github.com/go-kit/kit/log/json_logger.go new file mode 100644 index 0000000..66094b4 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/json_logger.go @@ -0,0 +1,89 @@ +package log + +import ( + "encoding" + "encoding/json" + "fmt" + "io" + "reflect" +) + +type jsonLogger struct { + io.Writer +} + +// NewJSONLogger returns a Logger that encodes keyvals to the Writer as a +// single JSON object. Each log event produces no more than one call to +// w.Write. The passed Writer must be safe for concurrent use by multiple +// goroutines if the returned Logger will be used concurrently. +func NewJSONLogger(w io.Writer) Logger { + return &jsonLogger{w} +} + +func (l *jsonLogger) Log(keyvals ...interface{}) error { + n := (len(keyvals) + 1) / 2 // +1 to handle case when len is odd + m := make(map[string]interface{}, n) + for i := 0; i < len(keyvals); i += 2 { + k := keyvals[i] + var v interface{} = ErrMissingValue + if i+1 < len(keyvals) { + v = keyvals[i+1] + } + merge(m, k, v) + } + return json.NewEncoder(l.Writer).Encode(m) +} + +func merge(dst map[string]interface{}, k, v interface{}) { + var key string + switch x := k.(type) { + case string: + key = x + case fmt.Stringer: + key = safeString(x) + default: + key = fmt.Sprint(x) + } + + // We want json.Marshaler and encoding.TextMarshaller to take priority over + // err.Error() and v.String(). But json.Marshall (called later) does that by + // default so we force a no-op if it's one of those 2 case. + switch x := v.(type) { + case json.Marshaler: + case encoding.TextMarshaler: + case error: + v = safeError(x) + case fmt.Stringer: + v = safeString(x) + } + + dst[key] = v +} + +func safeString(str fmt.Stringer) (s string) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() { + s = "NULL" + } else { + panic(panicVal) + } + } + }() + s = str.String() + return +} + +func safeError(err error) (s interface{}) { + defer func() { + if panicVal := recover(); panicVal != nil { + if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() { + s = nil + } else { + panic(panicVal) + } + } + }() + s = err.Error() + return +} diff --git a/vendor/github.com/go-kit/kit/log/level/doc.go b/vendor/github.com/go-kit/kit/log/level/doc.go new file mode 100644 index 0000000..feadc4c --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/level/doc.go @@ -0,0 +1,22 @@ +// Package level implements leveled logging on top of package log. To use the +// level package, create a logger as per normal in your func main, and wrap it +// with level.NewFilter. +// +// var logger log.Logger +// logger = log.NewLogfmtLogger(os.Stderr) +// logger = level.NewFilter(logger, level.AllowInfo()) // <-- +// logger = log.With(logger, "ts", log.DefaultTimestampUTC) +// +// Then, at the callsites, use one of the level.Debug, Info, Warn, or Error +// helper methods to emit leveled log events. +// +// logger.Log("foo", "bar") // as normal, no level +// level.Debug(logger).Log("request_id", reqID, "trace_data", trace.Get()) +// if value > 100 { +// level.Error(logger).Log("value", value) +// } +// +// NewFilter allows precise control over what happens when a log event is +// emitted without a level key, or if a squelched level is used. Check the +// Option functions for details. +package level diff --git a/vendor/github.com/go-kit/kit/log/level/level.go b/vendor/github.com/go-kit/kit/log/level/level.go new file mode 100644 index 0000000..dd4ef60 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/level/level.go @@ -0,0 +1,205 @@ +package level + +import "github.com/go-kit/kit/log" + +// Error returns a logger that includes a Key/ErrorValue pair. +func Error(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), ErrorValue()) +} + +// Warn returns a logger that includes a Key/WarnValue pair. +func Warn(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), WarnValue()) +} + +// Info returns a logger that includes a Key/InfoValue pair. +func Info(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), InfoValue()) +} + +// Debug returns a logger that includes a Key/DebugValue pair. +func Debug(logger log.Logger) log.Logger { + return log.WithPrefix(logger, Key(), DebugValue()) +} + +// NewFilter wraps next and implements level filtering. See the commentary on +// the Option functions for a detailed description of how to configure levels. +// If no options are provided, all leveled log events created with Debug, +// Info, Warn or Error helper methods are squelched and non-leveled log +// events are passed to next unmodified. +func NewFilter(next log.Logger, options ...Option) log.Logger { + l := &logger{ + next: next, + } + for _, option := range options { + option(l) + } + return l +} + +type logger struct { + next log.Logger + allowed level + squelchNoLevel bool + errNotAllowed error + errNoLevel error +} + +func (l *logger) Log(keyvals ...interface{}) error { + var hasLevel, levelAllowed bool + for i := 1; i < len(keyvals); i += 2 { + if v, ok := keyvals[i].(*levelValue); ok { + hasLevel = true + levelAllowed = l.allowed&v.level != 0 + break + } + } + if !hasLevel && l.squelchNoLevel { + return l.errNoLevel + } + if hasLevel && !levelAllowed { + return l.errNotAllowed + } + return l.next.Log(keyvals...) +} + +// Option sets a parameter for the leveled logger. +type Option func(*logger) + +// AllowAll is an alias for AllowDebug. +func AllowAll() Option { + return AllowDebug() +} + +// AllowDebug allows error, warn, info and debug level log events to pass. +func AllowDebug() Option { + return allowed(levelError | levelWarn | levelInfo | levelDebug) +} + +// AllowInfo allows error, warn and info level log events to pass. +func AllowInfo() Option { + return allowed(levelError | levelWarn | levelInfo) +} + +// AllowWarn allows error and warn level log events to pass. +func AllowWarn() Option { + return allowed(levelError | levelWarn) +} + +// AllowError allows only error level log events to pass. +func AllowError() Option { + return allowed(levelError) +} + +// AllowNone allows no leveled log events to pass. +func AllowNone() Option { + return allowed(0) +} + +func allowed(allowed level) Option { + return func(l *logger) { l.allowed = allowed } +} + +// ErrNotAllowed sets the error to return from Log when it squelches a log +// event disallowed by the configured Allow[Level] option. By default, +// ErrNotAllowed is nil; in this case the log event is squelched with no +// error. +func ErrNotAllowed(err error) Option { + return func(l *logger) { l.errNotAllowed = err } +} + +// SquelchNoLevel instructs Log to squelch log events with no level, so that +// they don't proceed through to the wrapped logger. If SquelchNoLevel is set +// to true and a log event is squelched in this way, the error value +// configured with ErrNoLevel is returned to the caller. +func SquelchNoLevel(squelch bool) Option { + return func(l *logger) { l.squelchNoLevel = squelch } +} + +// ErrNoLevel sets the error to return from Log when it squelches a log event +// with no level. By default, ErrNoLevel is nil; in this case the log event is +// squelched with no error. +func ErrNoLevel(err error) Option { + return func(l *logger) { l.errNoLevel = err } +} + +// NewInjector wraps next and returns a logger that adds a Key/level pair to +// the beginning of log events that don't already contain a level. In effect, +// this gives a default level to logs without a level. +func NewInjector(next log.Logger, level Value) log.Logger { + return &injector{ + next: next, + level: level, + } +} + +type injector struct { + next log.Logger + level interface{} +} + +func (l *injector) Log(keyvals ...interface{}) error { + for i := 1; i < len(keyvals); i += 2 { + if _, ok := keyvals[i].(*levelValue); ok { + return l.next.Log(keyvals...) + } + } + kvs := make([]interface{}, len(keyvals)+2) + kvs[0], kvs[1] = key, l.level + copy(kvs[2:], keyvals) + return l.next.Log(kvs...) +} + +// Value is the interface that each of the canonical level values implement. +// It contains unexported methods that prevent types from other packages from +// implementing it and guaranteeing that NewFilter can distinguish the levels +// defined in this package from all other values. +type Value interface { + String() string + levelVal() +} + +// Key returns the unique key added to log events by the loggers in this +// package. +func Key() interface{} { return key } + +// ErrorValue returns the unique value added to log events by Error. +func ErrorValue() Value { return errorValue } + +// WarnValue returns the unique value added to log events by Warn. +func WarnValue() Value { return warnValue } + +// InfoValue returns the unique value added to log events by Info. +func InfoValue() Value { return infoValue } + +// DebugValue returns the unique value added to log events by Warn. +func DebugValue() Value { return debugValue } + +var ( + // key is of type interfae{} so that it allocates once during package + // initialization and avoids allocating every time the value is added to a + // []interface{} later. + key interface{} = "level" + + errorValue = &levelValue{level: levelError, name: "error"} + warnValue = &levelValue{level: levelWarn, name: "warn"} + infoValue = &levelValue{level: levelInfo, name: "info"} + debugValue = &levelValue{level: levelDebug, name: "debug"} +) + +type level byte + +const ( + levelDebug level = 1 << iota + levelInfo + levelWarn + levelError +) + +type levelValue struct { + name string + level +} + +func (v *levelValue) String() string { return v.name } +func (v *levelValue) levelVal() {} diff --git a/vendor/github.com/go-kit/kit/log/log.go b/vendor/github.com/go-kit/kit/log/log.go new file mode 100644 index 0000000..66a9e2f --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/log.go @@ -0,0 +1,135 @@ +package log + +import "errors" + +// Logger is the fundamental interface for all log operations. Log creates a +// log event from keyvals, a variadic sequence of alternating keys and values. +// Implementations must be safe for concurrent use by multiple goroutines. In +// particular, any implementation of Logger that appends to keyvals or +// modifies or retains any of its elements must make a copy first. +type Logger interface { + Log(keyvals ...interface{}) error +} + +// ErrMissingValue is appended to keyvals slices with odd length to substitute +// the missing value. +var ErrMissingValue = errors.New("(MISSING)") + +// With returns a new contextual logger with keyvals prepended to those passed +// to calls to Log. If logger is also a contextual logger created by With or +// WithPrefix, keyvals is appended to the existing context. +// +// The returned Logger replaces all value elements (odd indexes) containing a +// Valuer with their generated value for each call to its Log method. +func With(logger Logger, keyvals ...interface{}) Logger { + if len(keyvals) == 0 { + return logger + } + l := newContext(logger) + kvs := append(l.keyvals, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + return &context{ + logger: l.logger, + // Limiting the capacity of the stored keyvals ensures that a new + // backing array is created if the slice must grow in Log or With. + // Using the extra capacity without copying risks a data race that + // would violate the Logger interface contract. + keyvals: kvs[:len(kvs):len(kvs)], + hasValuer: l.hasValuer || containsValuer(keyvals), + } +} + +// WithPrefix returns a new contextual logger with keyvals prepended to those +// passed to calls to Log. If logger is also a contextual logger created by +// With or WithPrefix, keyvals is prepended to the existing context. +// +// The returned Logger replaces all value elements (odd indexes) containing a +// Valuer with their generated value for each call to its Log method. +func WithPrefix(logger Logger, keyvals ...interface{}) Logger { + if len(keyvals) == 0 { + return logger + } + l := newContext(logger) + // Limiting the capacity of the stored keyvals ensures that a new + // backing array is created if the slice must grow in Log or With. + // Using the extra capacity without copying risks a data race that + // would violate the Logger interface contract. + n := len(l.keyvals) + len(keyvals) + if len(keyvals)%2 != 0 { + n++ + } + kvs := make([]interface{}, 0, n) + kvs = append(kvs, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + kvs = append(kvs, l.keyvals...) + return &context{ + logger: l.logger, + keyvals: kvs, + hasValuer: l.hasValuer || containsValuer(keyvals), + } +} + +// context is the Logger implementation returned by With and WithPrefix. It +// wraps a Logger and holds keyvals that it includes in all log events. Its +// Log method calls bindValues to generate values for each Valuer in the +// context keyvals. +// +// A context must always have the same number of stack frames between calls to +// its Log method and the eventual binding of Valuers to their value. This +// requirement comes from the functional requirement to allow a context to +// resolve application call site information for a Caller stored in the +// context. To do this we must be able to predict the number of logging +// functions on the stack when bindValues is called. +// +// Two implementation details provide the needed stack depth consistency. +// +// 1. newContext avoids introducing an additional layer when asked to +// wrap another context. +// 2. With and WithPrefix avoid introducing an additional layer by +// returning a newly constructed context with a merged keyvals rather +// than simply wrapping the existing context. +type context struct { + logger Logger + keyvals []interface{} + hasValuer bool +} + +func newContext(logger Logger) *context { + if c, ok := logger.(*context); ok { + return c + } + return &context{logger: logger} +} + +// Log replaces all value elements (odd indexes) containing a Valuer in the +// stored context with their generated value, appends keyvals, and passes the +// result to the wrapped Logger. +func (l *context) Log(keyvals ...interface{}) error { + kvs := append(l.keyvals, keyvals...) + if len(kvs)%2 != 0 { + kvs = append(kvs, ErrMissingValue) + } + if l.hasValuer { + // If no keyvals were appended above then we must copy l.keyvals so + // that future log events will reevaluate the stored Valuers. + if len(keyvals) == 0 { + kvs = append([]interface{}{}, l.keyvals...) + } + bindValues(kvs[:len(l.keyvals)]) + } + return l.logger.Log(kvs...) +} + +// LoggerFunc is an adapter to allow use of ordinary functions as Loggers. If +// f is a function with the appropriate signature, LoggerFunc(f) is a Logger +// object that calls f. +type LoggerFunc func(...interface{}) error + +// Log implements Logger by calling f(keyvals...). +func (f LoggerFunc) Log(keyvals ...interface{}) error { + return f(keyvals...) +} diff --git a/vendor/github.com/go-kit/kit/log/logfmt_logger.go b/vendor/github.com/go-kit/kit/log/logfmt_logger.go new file mode 100644 index 0000000..a003052 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/logfmt_logger.go @@ -0,0 +1,62 @@ +package log + +import ( + "bytes" + "io" + "sync" + + "github.com/go-logfmt/logfmt" +) + +type logfmtEncoder struct { + *logfmt.Encoder + buf bytes.Buffer +} + +func (l *logfmtEncoder) Reset() { + l.Encoder.Reset() + l.buf.Reset() +} + +var logfmtEncoderPool = sync.Pool{ + New: func() interface{} { + var enc logfmtEncoder + enc.Encoder = logfmt.NewEncoder(&enc.buf) + return &enc + }, +} + +type logfmtLogger struct { + w io.Writer +} + +// NewLogfmtLogger returns a logger that encodes keyvals to the Writer in +// logfmt format. Each log event produces no more than one call to w.Write. +// The passed Writer must be safe for concurrent use by multiple goroutines if +// the returned Logger will be used concurrently. +func NewLogfmtLogger(w io.Writer) Logger { + return &logfmtLogger{w} +} + +func (l logfmtLogger) Log(keyvals ...interface{}) error { + enc := logfmtEncoderPool.Get().(*logfmtEncoder) + enc.Reset() + defer logfmtEncoderPool.Put(enc) + + if err := enc.EncodeKeyvals(keyvals...); err != nil { + return err + } + + // Add newline to the end of the buffer + if err := enc.EndRecord(); err != nil { + return err + } + + // The Logger interface requires implementations to be safe for concurrent + // use by multiple goroutines. For this implementation that means making + // only one call to l.w.Write() for each call to Log. + if _, err := l.w.Write(enc.buf.Bytes()); err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/go-kit/kit/log/nop_logger.go b/vendor/github.com/go-kit/kit/log/nop_logger.go new file mode 100644 index 0000000..1047d62 --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/nop_logger.go @@ -0,0 +1,8 @@ +package log + +type nopLogger struct{} + +// NewNopLogger returns a logger that doesn't do anything. +func NewNopLogger() Logger { return nopLogger{} } + +func (nopLogger) Log(...interface{}) error { return nil } diff --git a/vendor/github.com/go-kit/kit/log/stdlib.go b/vendor/github.com/go-kit/kit/log/stdlib.go new file mode 100644 index 0000000..ff96b5d --- /dev/null +++ b/vendor/github.com/go-kit/kit/log/stdlib.go @@ -0,0 +1,116 @@ +package log + +import ( + "io" + "log" + "regexp" + "strings" +) + +// StdlibWriter implements io.Writer by invoking the stdlib log.Print. It's +// designed to be passed to a Go kit logger as the writer, for cases where +// it's necessary to redirect all Go kit log output to the stdlib logger. +// +// If you have any choice in the matter, you shouldn't use this. Prefer to +// redirect the stdlib log to the Go kit logger via NewStdlibAdapter. +type StdlibWriter struct{} + +// Write implements io.Writer. +func (w StdlibWriter) Write(p []byte) (int, error) { + log.Print(strings.TrimSpace(string(p))) + return len(p), nil +} + +// StdlibAdapter wraps a Logger and allows it to be passed to the stdlib +// logger's SetOutput. It will extract date/timestamps, filenames, and +// messages, and place them under relevant keys. +type StdlibAdapter struct { + Logger + timestampKey string + fileKey string + messageKey string +} + +// StdlibAdapterOption sets a parameter for the StdlibAdapter. +type StdlibAdapterOption func(*StdlibAdapter) + +// TimestampKey sets the key for the timestamp field. By default, it's "ts". +func TimestampKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.timestampKey = key } +} + +// FileKey sets the key for the file and line field. By default, it's "caller". +func FileKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.fileKey = key } +} + +// MessageKey sets the key for the actual log message. By default, it's "msg". +func MessageKey(key string) StdlibAdapterOption { + return func(a *StdlibAdapter) { a.messageKey = key } +} + +// NewStdlibAdapter returns a new StdlibAdapter wrapper around the passed +// logger. It's designed to be passed to log.SetOutput. +func NewStdlibAdapter(logger Logger, options ...StdlibAdapterOption) io.Writer { + a := StdlibAdapter{ + Logger: logger, + timestampKey: "ts", + fileKey: "caller", + messageKey: "msg", + } + for _, option := range options { + option(&a) + } + return a +} + +func (a StdlibAdapter) Write(p []byte) (int, error) { + result := subexps(p) + keyvals := []interface{}{} + var timestamp string + if date, ok := result["date"]; ok && date != "" { + timestamp = date + } + if time, ok := result["time"]; ok && time != "" { + if timestamp != "" { + timestamp += " " + } + timestamp += time + } + if timestamp != "" { + keyvals = append(keyvals, a.timestampKey, timestamp) + } + if file, ok := result["file"]; ok && file != "" { + keyvals = append(keyvals, a.fileKey, file) + } + if msg, ok := result["msg"]; ok { + keyvals = append(keyvals, a.messageKey, msg) + } + if err := a.Logger.Log(keyvals...); err != nil { + return 0, err + } + return len(p), nil +} + +const ( + logRegexpDate = `(?P[0-9]{4}/[0-9]{2}/[0-9]{2})?[ ]?` + logRegexpTime = `(?P