Initial commit, adding tests for etcd

tests/e2e_test_from_src_include.yml

@@ -0,0 +1,39 @@
+- hosts: localhost
+  tags:
+    - classic
+  tasks:
+    - include_tasks: tasks/update_os_vars.yml
+    - include_tasks: tasks/install_etcd_src.yml
+    - include_tasks: tasks/prepare_e2e.yml
+    - name: Run e2e test
+      shell: go test -timeout 10m -v ./e2e &> /var/tmp/test.log
+      args:
+        chdir: "{{ src_dir }}"
+      ignore_errors: true
+      environment:
+        GOPATH: "{{ src_dir }}/gopath:{{ go_path }}"
+        EXPECT_DEBUG: true
+    - always:
+      - name: Pull out the logs
+        fetch:
+          dest: "{{ artifacts }}/"
+          src: "/var/tmp/test.log"
+          flat: yes
+      - include_role:
+          name: remove_req_dir
+        vars:
+          path: "{{ item.path }}"
+          patterns: "{{ item.patterns }}"
+          file_type: directory
+        with_items:
+                - { path: "{{src_dir}}", patterns: ""}
+                - { path: "/var/tmp/cafile", patterns: ""}
+                - { path: "/tmp", patterns: "test*"}
+      - name: Check the results
+        shell: grep "^\-\-\- FAIL" "{{ artifacts}}"/test.log | awk '{print $3}'
+        register: failed_cases
+        ignore_errors: true
+      - name: Store failed cases
+        local_action: copy content={{ failed_cases.stdout }} dest={{ artifacts }}/failed_cases.log
+        when: failed_cases.stdout != ""
+        failed_when: failed_cases.stdout

tests/files/get_golang_pkgs.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python
+import logging
+import re
+import yum
+from optparse import OptionParser
+
+
+def get_pkg_name(req_line, rpm_list):
+    kws = re.split("[() /]", req_line.strip())[2:5]
+    kws.reverse()
+    compare_str = "devel"
+    tmp_list = rpm_list
+    for kw in kws:
+        tmp_list = [_ for _ in tmp_list if re.findall("%s" % compare_str, _)]
+        if len(tmp_list) > 1:
+            compare_str = "%s-%s" % (kw, compare_str)
+        else:
+            break
+
+    if not tmp_list:
+        if kws[-1] == "github.com":
+            compare_str = "github-.*-%s-devel" % kws[0]
+            tmp_list = [_ for _ in rpm_list if re.findall("%s" % compare_str, _)]
+        if kws[-1] == "golang.org":
+            compare_str = "golangorg-%s-devel" % kws[0]
+            tmp_list = [_ for _ in rpm_list if re.findall("%s" % compare_str, _)]
+        if kws[-1] == "google.golang.org":
+            compare_str = "-%s-.*-devel" % kws[1]
+            tmp_list = [_ for _ in rpm_list if re.findall("%s" % compare_str, _)]
+
+
+    return tmp_list
+
+
+if __name__ == '__main__':
+    parser = OptionParser()
+    parser.add_option("-f", "--spec_file", dest="sfile",
+                       default="")
+    
+    (options, args) = parser.parse_args()
+    sfile = options.sfile
+
+    yb = yum.YumBase()
+    yum_list = yb.doPackageLists()
+    alist = sorted(yum_list.available)
+    source_rpms = [_.name for _ in alist if re.match("golang", _.name)]
+
+    install_pkgs = []
+    with open(sfile) as f:
+        for line in f:
+            install_pkgs += get_pkg_name(line, source_rpms)
+    for pkg in set(install_pkgs):
+        print pkg

tests/group_vars/all.yml

@@ -0,0 +1,8 @@
+artifacts: ./artifacts
+rpm_base_url: https://kojipkgs.fedoraproject.org/packages
+go_path: /root/golang
+debug: false
+req_packages: ['git', 'golang', 'rpm-build', 'yum', 'libselinux-python']
+fedora_req_packages: ['go-compilers-golang-compiler']
+is_fedora: false
+is_rhel: false

tests/roles/create_ca_files_with_cfssl/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+
+- name: Create cafile directory
+  file:
+    path: "/var/tmp/cafile"
+    state: directory
+- name: Get cfssl from github
+  shell: go get -u github.com/cloudflare/cfssl/cmd/...
+- name:
+  template:
+    src: "{{ item }}.j2"
+    dest: "/var/tmp/cafile/{{ item }}"
+  with_items:
+    - ca-csr.json
+    - ca-config.json
+    - req-csr.json
+- name: Generate CA certificate 
+  shell: cfssl genkey -initca ca-csr.json | cfssljson -bare ca
+  args:
+    chdir: /var/tmp/cafile
+- name: Generate certificate and private key with CA
+  shell: cfssl gencert -ca ca.pem -ca-key ca-key.pem -config ca-config.json req-csr.json | cfssljson -bare {{ item }}
+  args:
+    chdir: /var/tmp/cafile
+  with_items:
+    - "{{ cert_names }}"
+- name: Copy needed certificates and private keys to destination
+  shell: cp -f /var/tmp/cafile/{{ item.src }} {{ src_dir }}/integration/fixtures/{{ item.dst }}
+  with_items:
+    - "{{ cert_srcs_dsts }}"

tests/roles/install_go_repo_from_github/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+
+- name: Create bin
+  file:
+    path: "{{ bin_dir }}"
+    state: directory
+
+- name: Create src code directory
+  file:
+    path: "{{ repo_path }}/{{ repo_name }}"
+    state: directory
+
+- name: Clone repo to /var/tmp
+  shell: git clone "{{ git_repo }}"
+  args:
+    chdir: "{{ repo_path }}"
+- name: Determine if use make 
+  shell: ls "{{ repo_path }}/{{ repo_name }}" | grep -i makefile
+  ignore_errors: true
+  register: output
+
+- name: Set compile with make
+  set_fact:
+    compile_method: make
+  when: output.rc == 0
+
+- name: Determine if use build
+  shell: ls "{{ repo_path }}/{{ repo_name }}" | grep -i build
+  ignore_errors: true
+  register: output
+- name: Set compile with build
+  set_fact:
+    compile_method: build
+  when: output.rc == 0
+
+- name: Build with make
+  shell: make
+  args:
+    chdir: "{{ repo_path }}/{{ repo_name }}"
+  when: "compile_method == 'make'"
+- name: Build with ./build
+  shell: ./build
+  args:
+    chdir: "{{ repo_path }}/{{ repo_name }}"
+  when: "compile_method == 'build'"
+- name: Determine if bin dir is generated
+  shell: ls "{{ repo_path }}/{{ repo_name }}"/bin
+  ignore_errors: true
+  register: output
+- name: Set compiled binary dir
+  set_fact:
+    c_bin_dir: "{{ repo_path }}/{{ repo_name }}/bin"
+  when: output.rc == 0
+- name: Set compiled binary dir
+  set_fact:
+    c_bin_dir: "{{ repo_path }}/{{ repo_name }}"
+  when: output.rc != 0
+- name: Copy generated binary to customer bin dir
+  shell: cp {{ c_bin_dir }}/{{ item }} {{ bin_dir }}
+  with_items:
+    "{{ compiled_binary }}"
+- name: Clean up source directory
+  file:
+    path: "{{ repo_path }}/{{ repo_name }}"
+    state: absent

tests/roles/install_golang_dep_from_spec/tasks/main.yml

@@ -0,0 +1,13 @@
+---
+
+- name: Get depended golang lines from spec
+  shell: "{% raw %} grep '^BuildRequires: golang' {% endraw %} /root/rpmbuild/SPECS/{{ package_name }}.spec | sort |uniq > /var/tmp/br"
+- name: Turned the spec lines into packages
+  shell: python "{{ role_path }}"/../../files/get_golang_pkgs.py -f /var/tmp/br
+  register: reqs
+- name: Install the packages required
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - "{{ reqs.stdout_lines }}"

tests/roles/install_src_rpm/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+
+- name: Get installed rpm information
+  yum:
+    list: "{{ package_name }}"
+  register: etcd_info_output
+- name: Set up src url
+  set_fact:
+    etcd_src_url: "{{ rpm_base_url }}/etcd/{{ item['version'] }}/{{ item['release'] }}/src/etcd-{{ item['version'] }}-{{ item['release'] }}.src.rpm"
+    package_path: "/var/tmp/{{ package_name }}.src.rpm"
+  when: item ['yumstate'] == 'installed'
+  with_items:
+    - "{{ etcd_info_output.results }}"
+- name: Remove the exist target file
+  file:
+    path: "{{ package_path }}"
+    state: absent
+- name: Download src package
+  get_url:
+    url: "{{ etcd_src_url }}"
+    dest: "{{ package_path }}"
+- name: Install src package with rpm
+  shell: rpm -ivh {{ package_path }}
+- name: Get commit from SPEC file
+  shell: awk '/^%global\scommit/{print $3}' /root/rpmbuild/SPECS/{{ package_name }}.spec
+  register: commit
+- name: Set src directory
+  set_fact:
+    src_dir: /root/rpmbuild/BUILD/{{ package_name }}-{{ commit.stdout}}

tests/roles/remove_req_dir/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+
+- name: Get files or directories fit for the patterns
+  find:
+    paths: "{{ path }}"
+    patterns: "{{ patterns }}"
+    file_type: "{{ file_type| default('file')}}"
+  register: remove_paths
+  when: (patterns is defined) and (patterns != "")
+- name: Remove the files or directories
+  file:
+    path: "{{ item_path.path }}"
+    state: absent
+  with_items:
+    - "{{ remove_paths.files }}"
+  loop_control:
+    loop_var: item_path
+  when: (patterns is defined) and (patterns != "")
+- name: Remove target files or diredctories
+  file:
+    path: "{{ path }}"
+    state: absent
+  when: (patterns is undefined) or (patterns == "")

tests/tasks/create_ca_files.yml

@@ -0,0 +1,5 @@
+---
+
+- hosts: localhost
+  tasks:
+    - name: Prepare the etcd-ca src

tests/tasks/install_etcd_src.yml

@@ -0,0 +1,19 @@
+---
+
+- name: Install request packages
+  package:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - "{{ req_packages }}"
+- include_role:
+    name: install_src_rpm
+  vars:
+    package_name: etcd
+- include_role:
+    name: install_golang_dep_from_spec
+  vars:
+    package_name: etcd
+  when: is_fedora
+- name: Install etcd source code
+  shell: rpmbuild -bp /root/rpmbuild/SPECS/etcd.spec

tests/tasks/prepare_e2e.yml

@@ -0,0 +1,59 @@
+---
+
+- name: Check Ca files
+  shell: openssl verify -CAfile ca.crt server.crt
+  args:
+    chdir: "{{ src_dir}}/integration/fixtures"
+  ignore_errors: true
+  register: ca_ok
+- name: Replace ca files
+  environment:
+    GOPATH: "{{ go_path }}"
+    PATH: "{{ ansible_env.PATH }}:{{ go_path }}/bin"
+  include_role:
+    name: create_ca_files_with_cfssl
+  vars:
+    cert_names: server
+    cert_srcs_dsts: [{ src: "ca.pem", dst: "ca.crt" }, { src: "server.pem", dst: "server.crt" }, { src: "server-key.pem", dst: "server.key.insecure" }]
+  when: ca_ok.rc != 0
+- name: Check if Godeps exist under src
+  find:
+    paths: "{{ src_dir }}"
+    patterns: Godeps
+    file_type: directory
+  register: godeps
+- name: Install glide
+  include_role:
+    name: install_go_repo_from_github
+  vars:
+    bin_dir: "{{ go_path }}/bin"
+    git_repo: https://github.com/Masterminds/glide.git
+    repo_path: "{{go_path}}/src/github.com/Masterminds"
+    repo_name: glide
+    compiled_binary: ['glide']
+  environment:
+    GOPATH: "{{ go_path }}"
+  when: godeps.matched == 0
+- name: Set up golang path
+  environment:
+    GOPATH: "{{ go_path }}"
+    PATH: "{{ ansible_env.PATH }}:{{ go_path }}/bin"
+    https_proxy: squid.redhat.com:3128
+  block:
+    - shell: mkdir gopath; ln -s {{ src_dir }}/Godeps/_workspace/src gopath/src; rm gopath/src/github.com/coreos/etcd; ln -s {{ src_dir }} gopath/src/github.com/coreos/etcd
+      args:
+        chdir: "{{ src_dir }}"
+      when: godeps.matched != 0
+    - shell: mkdir gopath; cp -r  vendor/* cmd/vendor/; ln -s {{ src_dir }}/cmd/vendor {{ src_dir }}/gopath/src; ln -s  {{ src_dir }} gopath/src/github.com/coreos/etcd
+      args:
+        chdir: "{{ src_dir }}"
+      when: godeps.matched == 0
+- name: Create bin under etcd src
+  file:
+    path: "{{ src_dir }}/bin"
+    state: directory
+- name: Copy exec file to src directory
+  shell: cp -f `which {{ item }}` {{ src_dir }}/bin/{{ item }}
+  with_items:
+    - etcd
+    - etcdctl

tests/tasks/update_os_vars.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Get distro name from /etc/os-release
+  raw: "grep '^NAME=' /etc/os-release | sed s'/NAME=//'"
+  register: distro
+- name: Set the is_fedora fact
+  set_fact:
+    is_fedora: true
+  when: "'Fedora' in distro.stdout"
+- name: Set the is_rhel fact
+  set_fact:
+    is_rhel: true
+  when: "'Red Hat Enterprise Linux' in distro.stdout"
+- name: Update request packages for Fedora
+  set_fact:
+    req_packages: "{{ req_packages }} + {{ fedora_req_packages }}"
+  when: is_fedora and fedora_req_packages is defined
+- name: Update request packages for RHEL
+  set_fact:
+    req_packages: "{{ req_packages }} + {{ rhel_req_packages }}"
+  when: is_rhel and rhel_req_packages is defined

tests/templates/ca-config.json.j2

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "8760h"
+    }
+  }
+}

tests/templates/ca-csr.json.j2

@@ -0,0 +1,16 @@
+{
+  "CN": "Autogenerated CA",
+  "key": {
+    "algo": "ecdsa",
+    "size": 384
+  },
+  "names": [
+    {
+      "O": "Honest Achmed's Used Certificates",
+      "OU": "Hastily-Generated Values Divison",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "US"
+    }
+  ]
+}

tests/templates/req-csr.json.j2

@@ -0,0 +1,17 @@
+{
+  "CN": "etcd",
+  "hosts": [
+    "localhost"
+  ],
+  "key": {
+    "algo": "ecdsa",
+    "size": 384
+  },
+  "names": [
+    {
+      "O": "autogenerated",
+      "OU": "etcd cluster",
+      "L": "the internet"
+    }
+  ]
+}

tests/tests.yml

@@ -0,0 +1,3 @@
+---
+# Tests for classic environment
+- include: e2e_test_from_src_include.yml