First public commit

19 files changed, 417 insertions, 0 deletions

diff --git a/test/integ-test/INTEGTEST.md b/test/integ-test/INTEGTEST.md
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/integ-test/INTEGTEST.md
diff --git a/test/integ-test/expected-results/configs-static-default.txt b/test/integ-test/expected-results/configs-static-default.txt
new file mode 100644
index 0000000..0aa8329
--- /dev/null
+++ b/test/integ-test/expected-results/configs-static-default.txt
@@ -0,0 +1,4 @@
+<pre>
+<a href="bootstrap.sh">bootstrap.sh</a>
+<a href="rc.local-bootstrap">rc.local-bootstrap</a>
+</pre>
diff --git a/test/integ-test/expected-results/ipxemenu.txt b/test/integ-test/expected-results/ipxemenu.txt
new file mode 100644
index 0000000..0560fcc
--- /dev/null
+++ b/test/integ-test/expected-results/ipxemenu.txt
@@ -0,0 +1,12 @@
+#!ipxe
+chain /poll/1/${netX/mac:hexhyp}
+menu Choose target to boot
+item /configs/coreos.ipxe coreos.ipxe
+item /env/production/configs/coreos.ipxe coreos.ipxe [production]
+
+choose target
+echo -n Enter hostname or none:
+read hostname
+set baseurl localhost:18888
+# Boot it as intended.
+chain ${target}
diff --git a/test/integ-test/expected-results/poll-k8s1-1.txt b/test/integ-test/expected-results/poll-k8s1-1.txt
new file mode 100644
index 0000000..46043fd
--- /dev/null
+++ b/test/integ-test/expected-results/poll-k8s1-1.txt
@@ -0,0 +1,14 @@
+#!ipxe
+
+set coreos-url http://stable.release.core-os.net/amd64-usr/1122.3.0
+
+echo This will currently autologin into tty1 on the console.
+echo From there you can su to root and install CoreOS to disk using:
+echo coreos-install -d /dev/sda -C stable
+echo You will probably need to chroot into /dev/sda9 to configure accounts.
+echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/
+
+kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1
+initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz
+
+boot
diff --git a/test/integ-test/expected-results/poll-k8s1-2.txt b/test/integ-test/expected-results/poll-k8s1-2.txt
new file mode 100644
index 0000000..46043fd
--- /dev/null
+++ b/test/integ-test/expected-results/poll-k8s1-2.txt
@@ -0,0 +1,14 @@
+#!ipxe
+
+set coreos-url http://stable.release.core-os.net/amd64-usr/1122.3.0
+
+echo This will currently autologin into tty1 on the console.
+echo From there you can su to root and install CoreOS to disk using:
+echo coreos-install -d /dev/sda -C stable
+echo You will probably need to chroot into /dev/sda9 to configure accounts.
+echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/
+
+kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1
+initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz
+
+boot
diff --git a/test/integ-test/expected-results/poll-k8s1-3-stg.txt b/test/integ-test/expected-results/poll-k8s1-3-stg.txt
new file mode 100644
index 0000000..46043fd
--- /dev/null
+++ b/test/integ-test/expected-results/poll-k8s1-3-stg.txt
@@ -0,0 +1,14 @@
+#!ipxe
+
+set coreos-url http://stable.release.core-os.net/amd64-usr/1122.3.0
+
+echo This will currently autologin into tty1 on the console.
+echo From there you can su to root and install CoreOS to disk using:
+echo coreos-install -d /dev/sda -C stable
+echo You will probably need to chroot into /dev/sda9 to configure accounts.
+echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/
+
+kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1
+initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz
+
+boot
diff --git a/test/integ-test/expected-results/poll-k8s1-4-stg.txt b/test/integ-test/expected-results/poll-k8s1-4-stg.txt
new file mode 100644
index 0000000..6195e9e
--- /dev/null
+++ b/test/integ-test/expected-results/poll-k8s1-4-stg.txt
@@ -0,0 +1,14 @@
+#!ipxe
+
+set coreos-url http://stable.release.core-os.net/amd64-usr/1298.6.0
+
+echo This will currently autologin into tty1 on the console.
+echo From there you can su to root and install CoreOS to disk using:
+echo coreos-install -d /dev/sda -C stable
+echo You will probably need to chroot into /dev/sda9 to configure accounts.
+echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/
+
+kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/env/staging/configs/coreos-baremetal.yaml?release=stable console=tty1 coreos.autologin=tty1
+initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz
+
+boot
diff --git a/test/integ-test/expected-results/poll-unknown-set-from-ui.txt b/test/integ-test/expected-results/poll-unknown-set-from-ui.txt
new file mode 100644
index 0000000..08e161e
--- /dev/null
+++ b/test/integ-test/expected-results/poll-unknown-set-from-ui.txt
@@ -0,0 +1,14 @@
+#!ipxe
+
+set coreos-url http://stable.release.core-os.net/amd64-usr/666.0
+
+echo This will currently autologin into tty1 on the console.
+echo From there you can su to root and install CoreOS to disk using:
+echo coreos-install -d /dev/sda -C stable
+echo You will probably need to chroot into /dev/sda9 to configure accounts.
+echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/
+
+kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://localhost:18888/configs/coreos-virtual.yaml?release=stable console=tty1 coreos.autologin=tty1
+initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz
+
+boot
diff --git a/test/integ-test/expected-results/poll-unknown.txt b/test/integ-test/expected-results/poll-unknown.txt
new file mode 100644
index 0000000..f17022f
--- /dev/null
+++ b/test/integ-test/expected-results/poll-unknown.txt
@@ -0,0 +1,2 @@
+#!ipxe
+prompt --key 0x02 --timeout 10000 shoelaces: Press Ctrl-B for manual override... && chain -ar http://localhost:18888/ipxemenu || chain -ar http://localhost:18888/poll/1/06-66-de-ad-be-ef
diff --git a/test/integ-test/expected-results/poll.txt b/test/integ-test/expected-results/poll.txt
new file mode 100644
index 0000000..dbba541
--- /dev/null
+++ b/test/integ-test/expected-results/poll.txt
@@ -0,0 +1,2 @@
+#!ipxe
+prompt --key 0x02 --timeout 10000 shoelaces: Press Ctrl-B for manual override... && chain -ar http://localhost:18888/ipxemenu || chain -ar http://localhost:18888/poll/1/ff-ff-ff-ff-ff-ff
diff --git a/test/integ-test/expected-results/rc.local-bootstrap b/test/integ-test/expected-results/rc.local-bootstrap
new file mode 100644
index 0000000..084ccc2
--- /dev/null
+++ b/test/integ-test/expected-results/rc.local-bootstrap
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+/usr/local/sbin/bootstrap > /var/log/bootstrap.log 2>&1 &
+exit 0
diff --git a/test/integ-test/expected-results/static.html b/test/integ-test/expected-results/static.html
new file mode 100644
index 0000000..eff16d6
--- /dev/null
+++ b/test/integ-test/expected-results/static.html
@@ -0,0 +1,7 @@
+<pre>
+<a href="css/">css/</a>
+<a href="fonts/">fonts/</a>
+<a href="img/">img/</a>
+<a href="js/">js/</a>
+<a href="templates/">templates/</a>
+</pre>
diff --git a/test/integ-test/integ-test-configs/env_overrides/production/ipxe/coreos.ipxe.slc b/test/integ-test/integ-test-configs/env_overrides/production/ipxe/coreos.ipxe.slc
new file mode 100644
index 0000000..f60b478
--- /dev/null
+++ b/test/integ-test/integ-test-configs/env_overrides/production/ipxe/coreos.ipxe.slc
@@ -0,0 +1,16 @@
+{{define "coreos.ipxe" -}}
+#!ipxe
+
+set coreos-url http://stable.release.core-os.net/amd64-usr/{{.version}}
+
+echo This will currently autologin into tty1 on the console.
+echo From there you can su to root and install CoreOS to disk using:
+echo coreos-install -d /dev/sda -C stable
+echo You will probably need to chroot into /dev/sda9 to configure accounts.
+echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/
+
+kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://{{.baseURL}}/configs/coreos-{{.cloudconfig}}.yaml?release=stable&hostname={{.hostname}} console=tty1 coreos.autologin=tty1
+initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz
+
+boot
+{{end}}
diff --git a/test/integ-test/integ-test-configs/env_overrides/staging/preseed/example.preseed.slc b/test/integ-test/integ-test-configs/env_overrides/staging/preseed/example.preseed.slc
new file mode 100644
index 0000000..824bcdd
--- /dev/null
+++ b/test/integ-test/integ-test-configs/env_overrides/staging/preseed/example.preseed.slc
@@ -0,0 +1,14 @@
+{{define "example.preseed" -}}
+
+d-i partman-auto-raid/recipe string \
+    1 4 0 ext3 /boot \
+        /dev/sda1#/dev/sdb1#/dev/sdc1#/dev/sdd1 \
+    . \
+    10 4 0 lvm - \
+        /dev/sda5#/dev/sdb5#/dev/sdc5#/dev/sdd5 \
+    .
+
+d-i partman-auto/disk string /dev/sda /dev/sdb /dev/sdc /dev/sdd
+d-i grub-installer/bootdev string /dev/sda /dev/sdb /dev/sdc /dev/sdd
+
+{{end}}
diff --git a/test/integ-test/integ-test-configs/ipxe/coreos.ipxe.slc b/test/integ-test/integ-test-configs/ipxe/coreos.ipxe.slc
new file mode 100644
index 0000000..7cb5156
--- /dev/null
+++ b/test/integ-test/integ-test-configs/ipxe/coreos.ipxe.slc
@@ -0,0 +1,16 @@
+{{define "coreos.ipxe" -}}
+#!ipxe
+
+set coreos-url http://stable.release.core-os.net/amd64-usr/{{.version}}
+
+echo This will currently autologin into tty1 on the console.
+echo From there you can su to root and install CoreOS to disk using:
+echo coreos-install -d /dev/sda -C stable
+echo You will probably need to chroot into /dev/sda9 to configure accounts.
+echo More info @ http://coreos.com/docs/running-coreos/bare-metal/installing-to-disk/
+
+kernel ${coreos-url}/coreos_production_pxe.vmlinuz cloud-config-url=http://{{.baseURL}}/configs/coreos-{{.cloudconfig}}.yaml?release=stable console=tty1 coreos.autologin=tty1
+initrd ${coreos-url}/coreos_production_pxe_image.cpio.gz
+
+boot
+{{end}}
diff --git a/test/integ-test/integ-test-configs/mappings.yaml b/test/integ-test/integ-test-configs/mappings.yaml
new file mode 100644
index 0000000..534b72f
--- /dev/null
+++ b/test/integ-test/integ-test-configs/mappings.yaml
@@ -0,0 +1,39 @@
+networkMaps:
+  - network: 20.20.20.20/24
+    script:
+      name: ubuntu-minimal.ipxe
+      params:
+        hostname: placeholder
+
+hostnameMaps:
+  - hostname: '(etcd|k8s)\d-m\d'
+    script:
+      name: coreos.ipxe
+      params:
+        version: 1122.3.0
+        cloudconfig: virtual
+  - hostname: '(etcd|k8s)\d-m\d'
+    script:
+      name: coreos.ipxe
+      params:
+        version: 1122.3.0
+        cloudconfig: virtual
+  - hostname: 'k8s1-4'
+    script:
+      name: coreos.ipxe
+      environment: staging
+      params:
+        version: 1298.6.0
+        cloudconfig: baremetal
+  - hostname: 'k8s1-\d'
+    script:
+      name: coreos.ipxe
+      params:
+        version: 1122.3.0
+        cloudconfig: baremetal
+  - hostname: 'k8s1-\d'
+    script:
+      name: coreos.ipxe
+      params:
+        version: 1122.3.0
+        cloudconfig: baremetal
diff --git a/test/integ-test/integ-test-configs/static/bootstrap.sh b/test/integ-test/integ-test-configs/static/bootstrap.sh
new file mode 100644
index 0000000..847ae99
--- /dev/null
+++ b/test/integ-test/integ-test-configs/static/bootstrap.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+export DEBIAN_FRONTEND=noninteractive
+
+echo Example boostrap configuration
+apt-get install hello
+
+echo '#!/bin/sh
+exit 0' > /etc/rc.local
+
+# Don't want to run this accidentally.
+chmod 0 /usr/local/sbin/bootstrap
diff --git a/test/integ-test/integ-test-configs/static/rc.local-bootstrap b/test/integ-test/integ-test-configs/static/rc.local-bootstrap
new file mode 100644
index 0000000..084ccc2
--- /dev/null
+++ b/test/integ-test/integ-test-configs/static/rc.local-bootstrap
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+/usr/local/sbin/bootstrap > /var/log/bootstrap.log 2>&1 &
+exit 0
diff --git a/test/integ-test/integ_test.py b/test/integ-test/integ_test.py
new file mode 100755
index 0000000..4f02fe3
--- /dev/null
+++ b/test/integ-test/integ_test.py
@@ -0,0 +1,216 @@
+#!/usr/bin/env python
+
+# Copyright 2018 ThousandEyes Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Test shoelaces """
+
+import os
+import signal
+import subprocess
+import sys
+import time
+import tempfile
+import string
+import pytest
+import requests
+import datetime
+import dateutil.parser
+from requests.exceptions import RequestException
+
+API_HOST = 'localhost'
+API_PORT = '18888'
+API_URL = "http://{}:{}".format(API_HOST, API_PORT)
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+BASE_DIR = os.path.dirname(os.path.dirname(TEST_DIR))
+FIXTURE_DIR = os.path.join(TEST_DIR, 'expected-results')
+STATIC_DIR = os.path.join(BASE_DIR, "web")
+SHOELACES_BINARY = os.path.join(BASE_DIR, "shoelaces")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def shoelaces_binary():
+    os.chdir(BASE_DIR)
+    subprocess.check_call(["go", "build"])
+    os.chdir(TEST_DIR)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def config_file(shoelaces_binary):
+    """ Create a temporary config file """
+    temp_config_tpl = string.Template("domain=$host\n"
+                                      "port=$port\n"
+                                      "data-dir=integ-test-configs\n"
+                                      "static-dir=$static_dir\n"
+                                      "template-extension=.slc\n"
+                                      "mappings-file=mappings.yaml\n"
+                                      "debug=true\n")
+    temp_config = temp_config_tpl.substitute(host=API_HOST,
+                                             port=API_PORT,
+                                             static_dir=STATIC_DIR)
+
+    sys.stderr.write("Using:\n{}".format(temp_config))
+    temp_cfg_file = tempfile.NamedTemporaryFile(delete=False)
+    temp_cfg_file.write(temp_config)
+    temp_cfg_file.flush()
+    temp_cfg_file_name = temp_cfg_file.name
+    temp_cfg_file.close()
+    yield temp_cfg_file_name
+    os.unlink(temp_cfg_file_name)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def shoelaces_instance(config_file):
+    """ Shoelaces test fixture. """
+    shoelaces_start_cmd = [SHOELACES_BINARY, "-config", config_file]
+    shoelaces = subprocess.Popen(shoelaces_start_cmd, preexec_fn=os.setsid)
+    sys.stderr.write("\nStarting Shoelaces...\n")
+    yield shoelaces
+    sys.stderr.write("\nShutting down Shoelaces...\n")
+    os.killpg(os.getpgid(shoelaces.pid), signal.SIGTERM)
+    sys.stderr.write("\nDone\n")
+
+
+def test_shoelaces_startup(shoelaces_instance):
+    """ Test API liveness """
+    attempts = 0
+    while True:
+        try:
+            req = requests.get('{}/'.format(API_URL))
+            req.raise_for_status()
+            sys.stderr.write('\n\nApi startup successful.\n')
+            break
+        except RequestException:
+            attempts += 1
+            if attempts > 10:
+                raise
+            sys.stderr.write(".")
+            time.sleep(1)
+
+
+@pytest.mark.parametrize(("path"), [("/"), ("/events"), ("/mappings")])
+def test_response_success(shoelaces_instance, path):
+    r = requests.get("{}{}".format(API_URL, path))
+    r.raise_for_status()
+
+
+REQUEST_RESPONSE_PAIRS = [("/static/", "static.html"),
+                          ("/configs/static/", "configs-static-default.txt"),
+                          ("/configs/static/rc.local-bootstrap",
+                           "rc.local-bootstrap"),
+                          ("/ipxemenu", "ipxemenu.txt")]
+
+
+@pytest.mark.parametrize(("request_path", "response_file"), REQUEST_RESPONSE_PAIRS)
+def test_request_response(shoelaces_instance, request_path, response_file):
+    with open(os.path.join(FIXTURE_DIR, response_file)) as response_body:
+        assert requests.get(
+            API_URL + request_path).text == response_body.read()
+
+
+def gen_mac_server_pairs():
+    generated = []
+    for m in range(0x00, 0x100, 0x11):
+        o = "{:02x}".format(m)
+        generated.append({'IP': '127.0.0.1', 'Mac': "ff:ff:ff:ff:ff:{}".format(o), 'Hostname': 'localhost'})
+        yield (o, list(generated))
+
+
+@pytest.mark.parametrize(("mac_last_octet", "servers"), gen_mac_server_pairs())
+def test_servers(shoelaces_instance, mac_last_octet, servers):
+    poll_url = "{}/poll/1/ff-ff-ff-ff-ff-{}".format(API_URL, mac_last_octet)
+    req = requests.get(poll_url)
+    req = requests.get("{}/ajax/servers".format(API_URL))
+    assert sorted(req.json()) == sorted(servers)
+
+
+def test_unknown_server(shoelaces_instance):
+    poll_url = "{}/poll/1/06-66-de-ad-be-ef".format(API_URL)
+    # Request for unknown host will give result in retries/polling
+    with open(os.path.join(FIXTURE_DIR, "poll-unknown.txt")) as poll:
+        assert requests.get(poll_url).text == poll.read()
+    # Setting the config for the new host should succeed.
+    requests.post(API_URL + '/update/target',
+                  {"target": "coreos.ipxe",
+                   "mac": "06:66:de:ad:be:ef",
+                   "version": "666.0",
+                   "cloudconfig": "virtual"}).raise_for_status()
+    # After setting we should be able to get the new config.
+    with open(os.path.join(FIXTURE_DIR, "poll-unknown-set-from-ui.txt")) as poll:
+        assert requests.get(poll_url).text == poll.read()
+    # Once fetched the host is now again "unknown"
+    with open(os.path.join(FIXTURE_DIR, "poll-unknown.txt")) as poll:
+        assert requests.get(poll_url).text == poll.read()
+
+
+def test_events(shoelaces_instance):
+    url = "{}/ajax/events".format(API_URL)
+    req = requests.get(url)
+    req.raise_for_status()
+    res = req.json()
+    # assert mac is in dictionary
+    assert '06:66:de:ad:be:ef' in res
+    # assert array with one element
+    assert isinstance(res['06:66:de:ad:be:ef'], list) and len(res['06:66:de:ad:be:ef']) == 4
+    # assert we have a date field
+    assert 'date' in res['06:66:de:ad:be:ef'][0]
+    # assert our date actually parses
+    assert dateutil.parser.parse(res['06:66:de:ad:be:ef'][0]['date'])
+    del res['06:66:de:ad:be:ef'][0]['date']
+    # compare to the expected result sans the date as it would be different
+    assert sorted(res['06:66:de:ad:be:ef'][0]) == sorted({'eventType': '0',
+                                                          'message': '0',
+                                                          'bootType': 'Manual',
+                                                          'server': {'mac':'',
+                                                                     'ip': '',
+                                                                     'hostname': '06-66-de-ad-be-ef'},
+                                                          'params': {'baseURL': 'localhost:18888',
+                                                                     'cloudconfig': 'virtual',
+                                                                     'hostname': '06-66-de-ad-be-ef',
+                                                                     'version': '666.0'},
+                                                          'script': 'coreos.ipxe'})
+
+
+POLL_PAIRS = [(None, "poll.txt"),
+              ({"host": "k8s1-3"}, "poll-k8s1-3-stg.txt"),
+              ({"host": "k8s1-4"}, "poll-k8s1-4-stg.txt"),
+              ({"host": "k8s1-1"}, "poll-k8s1-1.txt"),
+              ({"host": "k8s1-2"}, "poll-k8s1-2.txt")]
+
+
+@pytest.mark.parametrize(("params", "expected"), POLL_PAIRS)
+def test_poll(shoelaces_instance, params, expected):
+    """ Test Poll handler """
+    poll_url = "{}/poll/1/ff-ff-ff-ff-ff-ff".format(API_URL)
+    req = requests.get(poll_url, params=params)
+    req.raise_for_status()
+    with open(os.path.join(FIXTURE_DIR, expected), 'r') as poll:
+        assert poll.read() == req.text
+
+
+TPL_VARS_PAIRS = [("coreos.ipxe", "", ["cloudconfig", "version"]),
+                  ("coreos.ipxe", "default", ["cloudconfig", "version"]),
+                  ("coreos.ipxe", "production", ["cloudconfig", "version", "hostname"])]
+
+
+@pytest.mark.parametrize(("script", "env", "vars"), TPL_VARS_PAIRS)
+def test_template_variables_list(shoelaces_instance, script, env, vars):
+    url = "{}/ajax/script/params".format(API_URL)
+    req = requests.get(url, params={"script": script, "environment": env})
+    req.raise_for_status()
+    assert sorted(req.json()) == sorted(vars)
+
+
+if __name__ == "__main__":
+    pytest.main(args=['-v'], plugins=None)