Merge pull request #253 from pulp/commit-migrations

Commit migrations and update travis (get Travis passing)

Author: asmacdo

.gitignore

@@ -59,8 +59,3 @@ target/
 
 # PyCharm
 .idea
-
-# Django migrations
-# Ignored during early development to prevent accidental commits;
-# we'll want these to not be ignored when releasing pulp 3.0
-pulp_python/app/migrations

.travis.yml

@@ -4,30 +4,29 @@
 # bootstrap.py to update this file.
 #
 # For more info visit https://github.com/pulp/plugin_template
-
+---
 sudo: required
 # https://docs.travis-ci.com/user/trusty-ci-environment/
 dist: xenial
 language: python
 python:
-    # python versions used in el7 SCL & supported fedora
-    - "3.6"
-    - "3.7"
+  # Fedora has 3.6 available (python36.x86_64), but pulp container images
+  # with it are not being built or published yet.
+  # - "3.6"
+  - "3.7"
 env:
   matrix:
     - DB=postgres TEST=pulp
     - DB=postgres TEST=docs
-
-
 matrix:
   exclude:
-
     - python: '3.6'
       env: DB=postgres TEST=docs
   fast_finish: true
 services:
-    - postgresql
-    - redis-server
+  - postgresql
+  - redis-server
+  - docker
 addons:
   apt:
     packages:
@@ -40,40 +39,43 @@ install: .travis/install.sh
 before_script: .travis/before_script.sh
 script: .travis/script.sh
 after_failure:
-  - sh -c "cat ~/django_runserver.log"
-  - sh -c "cat ~/resource_manager.log"
-  - sh -c "cat ~/reserved_workers-1.log"
+  - http --timeout 30 --check-status --pretty format --print hb http://localhost:24817/pulp/api/v3/status/
+  - sudo docker images
+  - sudo kubectl logs -l name=pulp-operator -c ansible --tail=10000
+  - sudo kubectl logs -l name=pulp-operator -c operator --tail=10000
+  - sudo kubectl logs -l app=pulp-api --tail=50000
+  - sudo kubectl logs -l app=pulp-content --tail=10000
+  - sudo kubectl logs -l app=pulp-resource-manager --tail=10000
+  - sudo kubectl logs -l app=pulp-worker --tail=10000
 jobs:
   include:
-  - stage: deploy-plugin-to-pypi
-    script:  bash .travis/publish_plugin_pypi.sh
-    if: tag IS present
-
-  - stage: publish-daily-client-gem
-    script: bash .travis/publish_client_gem.sh
-    env:
-      - DB=postgres
-      - TEST=bindings
-    if: type = cron
-  - stage: publish-daily-client-pypi
-    script: bash .travis/publish_client_pypi.sh
-    env:
-      - DB=postgres
-      - TEST=bindings
-    if: type = cron
-  - stage: publish-client-gem
-    script: bash .travis/publish_client_gem.sh
-    env:
-      - DB=postgres
-      - TEST=bindings
-    if: tag IS present
-  - stage: publish-client-pypi
-    script: bash .travis/publish_client_pypi.sh
-    env:
-      - DB=postgres
-      - TEST=bindings
-    if: tag IS present
-
+    - stage: deploy-plugin-to-pypi
+      install: skip
+      script: bash .travis/publish_plugin_pypi.sh
+      if: tag IS present
+    - stage: publish-daily-client-gem
+      script: bash .travis/publish_client_gem.sh
+      env:
+        - DB=postgres
+        - TEST=bindings
+      if: type = cron
+    - stage: publish-daily-client-pypi
+      script: bash .travis/publish_client_pypi.sh
+      env:
+        - DB=postgres
+        - TEST=bindings
+      if: type = cron
+    - stage: publish-client-gem
+      script: bash .travis/publish_client_gem.sh
+      env:
+        - DB=postgres
+        - TEST=bindings
+      if: tag IS present
+    - stage: publish-client-pypi
+      script: bash .travis/publish_client_pypi.sh
+      env:
+        - DB=postgres
+        - TEST=bindings
+      if: tag IS present
 notifications: None
-
-
+...

.travis/before_install.sh

@@ -24,8 +24,10 @@ export PULP_PLUGIN_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https
 export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
 export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
 export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
+export PULP_OPERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-operator\/pull\/(\d+)' | awk -F'/' '{print $7}')
 
-# dev_requirements should not be needed for testing; don't install them to make sure
+# test_requirements contains tools needed for flake8, etc.
+# So install them here rather than in install.sh
 pip install -r test_requirements.txt
 
 # check the commit message
@@ -45,6 +47,16 @@ if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
   cd ..
 fi
 
+
+git clone --depth=1 https://github.com/pulp/pulp-operator.git
+if [ -n "$PULP_OPERATOR_PR_NUMBER" ]; then
+  cd pulp-operator
+  git fetch --depth=1 origin +refs/pull/$PULP_OPERATOR_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  cd ..
+fi
+
+
 git clone --depth=1 https://github.com/pulp/pulpcore.git
 
 if [ -n "$PULP_PR_NUMBER" ]; then
@@ -65,19 +77,19 @@ if [ -n "$PULP_PLUGIN_PR_NUMBER" ]; then
 fi
 
 
+git clone --depth=1 https://github.com/PulpQE/pulp-smash.git
+
 if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
-  git clone --depth=1 https://github.com/PulpQE/pulp-smash.git
   cd pulp-smash
   git fetch --depth=1 origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
   git checkout FETCH_HEAD
   cd ..
 fi
 
-psql -c 'CREATE DATABASE pulp OWNER travis;'
+# pulp-smash already got installed via test_requirements.txt
+pip install --upgrade --force-reinstall ./pulp-smash
 
 pip install ansible
-cp pulp_python/.travis/playbook.yml ansible-pulp/playbook.yml
-cp pulp_python/.travis/postgres.yml ansible-pulp/postgres.yml
 
 cd pulp_python
 

.travis/before_script.sh

@@ -18,7 +18,12 @@ fi
 
 
 mkdir -p ~/.config/pulp_smash
-cp ../pulpcore/.travis/pulp-smash-config.json ~/.config/pulp_smash/settings.json
+
+if [ -f .travis/pulp-smash-config.json ]; then
+    sed "s/localhost/$(hostname)/g" .travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
+else
+    sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
+fi
 
 
 if [ -f $POST_BEFORE_SCRIPT ]; then

.travis/install.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 
 # WARNING: DO NOT EDIT!
 #
@@ -10,14 +10,108 @@
 set -euv
 
 if [ "$TEST" = 'docs' ]; then
-  pip3 install -r doc_requirements.txt
+
+  pip install -r ../pulpcore/doc_requirements.txt
+
+  pip install -r doc_requirements.txt
 fi
 
 pip install -r test_requirements.txt
 
-# Run Ansible playbook
-cd ../ansible-pulp
-ansible-galaxy install -r requirements.yml
+cd $TRAVIS_BUILD_DIR/../pulpcore/containers/
+
+# Although the tag name is not used outside of this script, we might use it
+# later. And it is nice to have a friendly identifier for it.
+# So we use the branch preferably, but need to replace the "/" with the valid
+# character "_" .
+#
+# Note that there are lots of other valid git branch name special characters
+# that are invalid in image tag names. To try to convert them, this would be a
+# starting point:
+# https://stackoverflow.com/a/50687120
+#
+# If we are on a PR
+if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]; then
+  TAG=$(echo $TRAVIS_PULL_REQUEST_BRANCH | tr / _)
+# For push builds, tag builds, and hopefully cron builds
+elif [ -n "$TRAVIS_BRANCH" ]; then
+  TAG=$(echo $TRAVIS_BRANCH | tr / _)
+  if [ "$TAG" = "master" ]; then
+    TAG=latest
+  fi
+else
+  # Fallback
+  TAG=$(git rev-parse --abbrev-ref HEAD | tr / _)
+fi
+
+
+PLUGIN=pulp_python
+
+
+# For pulpcore, and any other repo that might check out a pulp-certguard PR
+if [ -e $TRAVIS_BUILD_DIR/../pulp-certguard ]; then
+  PULP_CERTGUARD=./pulp-certguard
+else
+  # Otherwise, master branch release
+  PULP_CERTGUARD=git+https://github.com/pulp/pulp-certguard.git
+fi
+
+cat > vars/vars.yaml << VARSYAML
+---
+images:
+  - ${PLUGIN}-${TAG}:
+      image_name: $PLUGIN
+      tag: $TAG
+      pulpcore: ./pulpcore
+      pulpcore_plugin: ./pulpcore-plugin
+      plugins:
+        - $PULP_CERTGUARD
+        - ./$PLUGIN
+VARSYAML
+
+ansible-playbook build.yaml
+
+cd $TRAVIS_BUILD_DIR/../pulp-operator
+# Tell pulp-perator to deploy our image
+cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML
+apiVersion: pulpproject.org/v1alpha1
+kind: Pulp
+metadata:
+  name: example-pulp
+spec:
+  pulp_file_storage:
+    # k3s local-path requires this
+    access_mode: "ReadWriteOnce"
+    # We have a little over 40GB free on Travis VMs/instances
+    size: "40Gi"
+  image: $PLUGIN
+  tag: $TAG
+  database_connection:
+    username: pulp
+    password: pulp
+    admin_password: pulp
+  content_host: $(hostname):24816
+CRYAML
+
+# Install k3s, lightweight Kubernetes
+.travis/k3s-install.sh
+# Deploy pulp-operator, with the pulp containers, according to CRYAML
+sudo ./up.sh
+
+# Needed for the script below
+# Since it is being run during install rather than actual tests (unlike in
+# pulp-operator), and therefore does not trigger the equivalent after_failure
+# travis commands.
+show_logs_and_return_non_zero() {
+    readonly local rc="$?"
+
+    for containerlog in "pulp-api" "pulp-content" "pulp-resource-manager" "pulp-worker"
+    do
+      echo -en "travis_fold:start:$containerlog"'\\r'
+      sudo kubectl logs -l app=$containerlog --tail=10000
+      echo -en "travis_fold:end:$containerlog"'\\r'
+    done
 
-ansible-playbook --connection=local --inventory 127.0.0.1, playbook.yml --extra-vars \
-  "pulp_python_interpreter=$VIRTUAL_ENV/bin/python, pulp_install_dir=$VIRTUAL_ENV"
+    return "${rc}"
+}
+.travis/pulp-operator-check-and-wait.sh || show_logs_and_return_non_zero