Merge branch 'main' into sshj

Author: aphyr

README.md

@@ -180,3 +180,10 @@ this is because (I think) SSHJ tries to verify only the ed25519 key and
 ssh-keyscan -t ed25519 n1 >> ~/.ssh/known_hosts
 ...
 ```
+
+## Other Projects
+
+Additional projects that may be of interest:
+
+- [Jecci](https://github.com/michaelzenz/jecci): A wrapper framework around Jepsen
+- [Porcupine](https://github.com/anishathalye/porcupine): a linearizability checker written in Go.

doc/lxc.md

@@ -1,22 +1,11 @@
 # How to set up nodes via LXC
-## Debian Testing:
 
-(refer to https://wiki.debian.org/LXC)
+## Debian Buster
 
-```sh
-aptitude install lxc bridge-utils ebtables libvirt-bin debootstrap dnsmasq
-```
-
-Add this line to /etc/fstab:
+(Refer to https://wiki.debian.org/LXC)
 
 ```
-cgroup  /sys/fs/cgroup  cgroup  defaults  0   0
-```
-
-Mount ze cgroup
-
-```
-mount /sys/fs/cgroup
+apt install lxc debootstrap bridge-utils libvirt-clients libvirt-daemon-system iptables ebtables dnsmasq-base libxml2-utils iproute2
 ```
 
 Apply google and kernel parameters until checkconfig passes:
@@ -25,157 +14,112 @@ Apply google and kernel parameters until checkconfig passes:
 lxc-checkconfig
 ```
 
-Create a VM or five
+Create VMs:
 
 ```
-lxc-create -n n1 -t debian -- --release jessie
-lxc-create -n n2 -t debian -- --release jessie
-lxc-create -n n3 -t debian -- --release jessie
-lxc-create -n n4 -t debian -- --release jessie
-lxc-create -n n5 -t debian -- --release jessie
+for i in {1..10}; do sudo lxc-create -n n$i -t debian -- --release buster; done
 ```
 
-Note the root passwords.
-
-Edit /var/lib/lxc/n1/config and friends, changing the network hwaddr to something unique. I suggest using sequential mac addresses for n1, n2, n3, ....
+Add network configuration to each node. We assign each a sequential MAC
+address.
 
 ```
-# Template used to create this container: /usr/share/lxc/templates/lxc-debian
-# Parameters passed to the template:
-# For additional config options, please look at lxc.conf(5)
-
-lxc.rootfs = /var/lib/lxc/n1/rootfs
+for i in {1..10}; do
+sudo cat >>/var/lib/lxc/n${i}/config <<EOF
 
-# Common configuration
-lxc.include = /usr/share/lxc/config/debian.common.conf
-
-# Container specific configuration
-lxc.mount = /var/lib/lxc/n1/fstab
-lxc.utsname = n1
-lxc.arch = amd64
-
-# Stuff to add:
-lxc.network.type = veth
-lxc.network.flags = up
-lxc.network.link = virbr0
-lxc.network.ipv4 = 0.0.0.0/24
-lxc.network.hwaddr = 00:1E:62:AA:AA:AA
+# Network config
+lxc.net.0.type = veth
+lxc.net.0.flags = up
+lxc.net.0.link = virbr0
+lxc.net.0.hwaddr = 00:1E:62:AA:AA:$(printf "%02x" $i)
+EOF
+done
 ```
 
-Set up libvirt network, and assign MAC->IP bindings for the LXC node mac addrs
+Set up the virsh network bindings mapping those MAC addresses to hostnames and
+IP addresses:
 
-```sh
-virsh net-edit default
 ```
-
-```xml
-<network>
-  <name>default</name>
-  <uuid>08063db9-38f4-4c9c-8887-08000f13ce80</uuid>
-  <forward mode='nat'/>
-  <bridge name='virbr0' stp='on' delay='0'/>
-  <mac address='52:54:00:8e:29:d2'/>
-  <ip address='192.168.122.1' netmask='255.255.255.0'>
-    <dhcp>
-      <range start='192.168.122.11' end='192.168.122.100'/>
-      <host mac='00:1E:62:AA:AA:AA' name='n1' ip='192.168.122.11'/>
-      <host mac='00:1E:62:AA:AA:AB' name='n2' ip='192.168.122.12'/>
-      <host mac='00:1E:62:AA:AA:AC' name='n3' ip='192.168.122.13'/>
-      <host mac='00:1E:62:AA:AA:AD' name='n4' ip='192.168.122.14'/>
-      <host mac='00:1E:62:AA:AA:AE' name='n5' ip='192.168.122.15'/>
-    </dhcp>
-  </ip>
-</network>
+for i in {1..10}; do
+  virsh net-update --current default add-last ip-dhcp-host "<host mac=\"00:1E:62:AA:AA:$(printf "%02x" $i)\" name=\"n${i}\" ip=\"192.168.122.1$(printf "%02d" $i)\"/>"
+done
 ```
 
-Drop an entry in `/etc/resolv.conf` to read from the libvirt network dns:
+Start the network, and set it to start at boot so the dnsmasq will be
+available.
 
 ```
-nameserver 192.168.122.1  # Local libvirt dnsmasq
-nameserver 192.168.1.1    # Regular network resolver
+virsh net-autostart default;
+virsh net-start default
 ```
 
-Kill the system default dnsmasq (if you have one), and start the network (which
-in turn will start a replacement dnsmasq with the LXC config. Then, start up
-all the nodes. I have this in a bash script called `jepsen-start`:
+If you configure resolv.conf by hand, add the libvirt local dnsmasq to
+resolv.conf:
 
-```sh
-#!/bin/sh
-sudo service dnsmasq stop
-sudo virsh net-start default
-sudo lxc-start -d -n n1
-sudo lxc-start -d -n n2
-sudo lxc-start -d -n n3
-sudo lxc-start -d -n n4
-sudo lxc-start -d -n n5
 ```
-
-Fire up each VM:
-
-```sh
-jepsen-start
+echo -e "nameserver 192.168.122.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf
 ```
 
-Log into the containers, (may have to specify tty 0 to use console correctly) e.g.,:
+If you're letting dhclient manage it, then:
 
-```sh
-lxc-console --name n1 -t 0
 ```
-
-(Optional?) In the containers, update keys used by apt to verify packages:
-
-```sh
-apt-key update
-apt-get update
+echo "prepend domain-name-servers 192.168.122.1;" >>/etc/dhcp/dhclient.conf
+sudo service networking restart
 ```
 
-And set your root password--I use `root`/`root` by default in Jepsen.
+Slip your preferred SSH key into each node's `.authorized-keys`:
 
-```sh
-passwd
 ```
-
-Copy your SSH key (on host):
-
-```sh
-cat ~/.ssh/id_rsa.pub
+for i in {1..10}; do
+  mkdir -p /var/lib/lxc/n${i}/rootfs/root/.ssh
+  chmod 700 /var/lib/lxc/n${i}/rootfs/root/.ssh/
+  cp ~/.ssh/id_rsa.pub /var/lib/lxc/n${i}/rootfs/root/.ssh/authorized_keys
+  chmod 644 /var/lib/lxc/n${i}/rootfs/root/.ssh/authorized_keys
+done
 ```
 
-and add it to root's `authorized_keys` (in containers):
+And start the nodes:
 
-```sh
-apt-get install -y sudo vim
-mkdir ~/.ssh
-chmod 700 ~/.ssh
-touch ~/.ssh/authorized_keys
-chmod 600 ~/.ssh/authorized_keys
-vim ~/.ssh/authorized_keys
 ```
-
-Enable password-based login for root (used by jsch):
-```sh
-sed  -i 's,^PermitRootLogin .*,PermitRootLogin yes,g' /etc/ssh/sshd_config
-systemctl restart sshd
-apt install sudo
+for i in {1..10}; do
+  lxc-start -d -n n$i
+done
 ```
 
-[Remove systemd](http://without-systemd.org/wiki/index.php/How_to_remove_systemd_from_a_Debian_jessie/sid_installation). After you install sysvinit-core and sysvinit-utils, you may have to restart the container with /lib/sysvinit/init argument to lxc-start before apt will allow you to remove systemd.
+To stop them:
 
-Detach from the container with Control+a q, and repeat for the remaining nodes.
+```
+for i in {1..10}; do
+  lxc-stop -n n$i
+done
+```
 
-On the control node, drop entries in `~/.ssh/config` for nodes:
+Reset the root passwords to whatever you like. Jepsen uses `root` by default,
+and allow root logins with passwords on each container. If you've got an SSH
+agent set up, Jepsen can use that instead.
 
 ```
-Host n*
-User root
+for i in {1..10}; do
+  lxc-attach -n n${i} -- bash -c 'echo -e "root\nroot\n" | passwd root';
+  lxc-attach -n n${i} -- sed -i 's,^#\?PermitRootLogin .*,PermitRootLogin yes,g' /etc/ssh/sshd_config;
+  lxc-attach -n n${i} -- systemctl restart sshd;
+done
 ```
 
 Store the host keys unencrypted so that jsch can use them. If you already have
 the host keys, they may be unreadable to Jepsen--remove them from .known_hosts
 and rescan.
 
 ```
-for n in $(seq 1 5); do ssh-keyscan -t rsa n$n; done >> ~/.ssh/known_hosts
+for n in {1..10}; do ssh-keyscan -t rsa n$n; done >> ~/.ssh/known_hosts
+```
+
+Install `sudo`:
+
+```sh
+for i in {1..10}; do
+  lxc-attach -n n${i} -- apt install -y sudo
+done
 ```
 
 And check that you can SSH to the nodes
@@ -184,8 +128,6 @@ And check that you can SSH to the nodes
 cssh n1 n2 n3 n4 n5
 ```
 
-And that should mostly do it, I think.
-
 ## Ubuntu 14.04 / trusty
 
 Follow generally the same steps as for Debian, but the process is easier. Reference: https://help.ubuntu.com/lts/serverguide/lxc.html

doc/plan.md

@@ -4,6 +4,7 @@
 
 - Knossos: Better error messages when users pass models that fail on the
   first op (I think there's a ticket about this? Null pointer exception for i?)
+- When users enter a node multiple times into :nodes, complain early
 
 ## Visualizations
 

doc/tutorial/03-client.md

@@ -36,7 +36,7 @@ value in for the completion op.
 
 Now we need to take these operations and *apply* them to etcd. We'll use
 the [Verschlimmbessergung](https://github.com/aphyr/verschlimmbesserung)
-library to talk to etcd. We'll start by requireing Verschlimmbesserung, and
+library to talk to etcd. We'll start by requiring Verschlimmbesserung, and
 writing an empty implementation of Jepsen's Client protocol:
 
 ```clj

docker/.gitignore

@@ -1,3 +1,3 @@
 secret/*
 !.gitkeep
-
+./docker-compose.yml

docker/README.md

@@ -5,7 +5,9 @@ It is intended to be used by a CI tool or anyone with Docker who wants to try Je
 
 It contains all the jepsen dependencies and code. It uses [Docker
 Compose](https://github.com/docker/compose) to spin up the five containers used
-by Jepsen.
+by Jepsen. A script builds a `docker-compose.yml` file out of fragments in
+`template/`, because this is the future, and using `awk` to generate YAML to
+generate computers is *cloud native*.
 
 ## Quickstart
 
@@ -26,6 +28,8 @@ results.
 
 ## Advanced
 
+You can change the number of DB nodes by running (e.g.) `bin/up -n 9`.
+
 If you need to log into a DB node (e.g. to debug a test), you can `ssh n1` (or n2, n3, ...) from inside the control node, or:
 
 ```

docker/bin/build-docker-compose

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Builds a docker-compose file. You'd THINK we could do this with `replicas`
+# but nooooooo, down that path lies madness. Instead we're going to do some
+# janky templating with sed and awk. I am so, so sorry.
+
+# Takes a number of nodes to generate a file for, and emits a file
+# `docker-compose.yml`.
+
+NODE_COUNT=$1
+
+DEPS=""
+DBS=""
+
+# For each node
+for ((n=1;n<=NODE_COUNT;n++)); do
+  # Build up deps for control
+  LINE=`cat template/depends.yml | sed s/%%N%%/${n}/g`
+  DEPS="${DEPS}${LINE}"$'\n'
+
+  # Build up DB service
+  DB=`cat template/db.yml | sed s/%%N%%/${n}/g`
+  DBS="${DBS}${DB}"$'\n'
+done
+
+# Build docker-compose file
+export DEPS
+export DBS
+cat template/docker-compose.yml |
+  awk ' {gsub(/%%DEPS%%/, ENVIRON["DEPS"]); print} ' |
+  awk ' {gsub(/%%DBS%%/, ENVIRON["DBS"]); print} ' \
+  > docker-compose.yml

docker/bin/up

@@ -32,6 +32,7 @@ pushd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.."
 
 HELP=0
 INIT_ONLY=0
+NODE_COUNT=5
 DEV=""
 COMPOSE=${COMPOSE:-""}
 RUN_AS_DAEMON=0
@@ -70,6 +71,11 @@ do
             RUN_AS_DAEMON=1
             shift # past argument
             ;;
+        -n|--node-count)
+            NODE_COUNT=$2
+            shift
+            shift
+            ;;
         *)
             POSITIONAL+=("$1")
             ERROR "unknown option $1"
@@ -116,6 +122,10 @@ else
     INFO "No need to generate key pair"
 fi
 
+
+## Build dockerfile
+bin/build-docker-compose $NODE_COUNT
+
 # Make sure folders referenced in control Dockerfile exist and don't contain leftover files
 rm -rf ./control/jepsen
 mkdir -p ./control/jepsen/jepsen
@@ -141,17 +151,21 @@ exists docker-compose ||
 
 INFO "Running \`docker-compose build\`"
 # shellcheck disable=SC2086
-docker-compose -f docker-compose.yml ${COMPOSE} ${DEV} build
+docker-compose --compatibility -p jepsen -f docker-compose.yml ${COMPOSE} ${DEV} build
+
+# We need a fresh share volume each time we start, so we have a correct set of
+# DB hosts. Why does Docker make sharing state SO hard
+docker run --rm -v jepsen_jepsen-shared:/data/ debian:buster rm /data/nodes || true
 
 INFO "Running \`docker-compose up\`"
 if [ "${RUN_AS_DAEMON}" -eq 1 ]; then
     # shellcheck disable=SC2086
-    docker-compose -f docker-compose.yml ${COMPOSE} ${DEV} up -d
+    docker-compose --compatibility -p jepsen -f docker-compose.yml ${COMPOSE} ${DEV} up -d
     INFO "All containers started! Run \`docker ps\` to view, and \`bin/console\` to get started."
 else
     INFO "Please run \`bin/console\` in another terminal to proceed"
     # shellcheck disable=SC2086
-    docker-compose -f docker-compose.yml ${COMPOSE} ${DEV} up
+    docker-compose --compatibility -p jepsen -f docker-compose.yml ${COMPOSE} ${DEV} up
 fi
 
 popd