Add smoke tests for development environment

Signed-off-by: Paulo Gomes <paulo.gomes@weave.works>

Author: Paulo Gomes

Dockerfile.test

@@ -1,96 +1,28 @@
-# This Dockerfile tests the hack/Makefile output against git2go.
 ARG BASE_VARIANT=alpine
-ARG GO_VERSION=1.17.6
-ARG XX_VERSION=1.1.0
 
-FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
+ARG LIBGIT2_IMG
+ARG LIBGIT2_TAG
 
-FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
+FROM --platform=linux/amd64 ${LIBGIT2_IMG}:${LIBGIT2_TAG} as build-amd64
+FROM --platform=linux/arm64 ${LIBGIT2_IMG}:${LIBGIT2_TAG} as build-arm64
+FROM --platform=linux/arm/v7 ${LIBGIT2_IMG}:${LIBGIT2_TAG} as build-armv7
 
-FROM gostable AS go-linux
+FROM --platform=$BUILDPLATFORM build-$TARGETARCH$TARGETVARIANT AS build
 
-FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-deps
-
-RUN apk add --no-cache \
-        bash \
-        curl \
-        build-base \
-        linux-headers \
-        perl \
-        cmake \
-        pkgconfig \
-        gcc \
-        musl-dev \
-        clang \
-        lld
-
-COPY --from=xx / /
+RUN apk add clang lld pkgconfig ca-certificates
 
 ARG TARGETPLATFORM
 
+# Some dependencies have to installed 
+# for the target platform: https://github.com/tonistiigi/xx#go--cgo
 RUN xx-apk add --no-cache \
-        xx-c-essentials 
-
-RUN xx-apk add --no-cache \
-        xx-cxx-essentials 
-
-ARG TARGETPLATFORM
-RUN xx-apk add --no-cache \
-        build-base \
-        pkgconfig \
-        gcc \
-        musl-dev \
-        clang \
-        lld \
-        llvm \
-        linux-headers
-
-WORKDIR /build
-COPY hack/static.sh .
-
-ARG TARGETPLATFORM
-ENV CC=xx-clang
-ENV CXX=xx-clang++
-
-RUN CHOST=$(xx-clang --print-target-triple) \
-    ./static.sh build_libz
-
-RUN CHOST=$(xx-clang --print-target-triple) \
-    ./static.sh build_openssl
-
-RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \
-    export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \
-    export OPENSSL_ROOT_DIR="/usr/local/$(xx-info triple)" && \
-    export OPENSSL_CRYPTO_LIBRARY="/usr/local/$(xx-info triple)/lib64" && \
-    export OPENSSL_INCLUDE_DIR="/usr/local/$(xx-info triple)/include/openssl"
-
-RUN ./static.sh build_libssh2
-RUN ./static.sh build_libgit2
-
-
-FROM go-${TARGETOS} AS build
-
-# Copy cross-compilation tools
-COPY --from=xx / /
-# Copy compiled libraries
-COPY --from=build-deps /usr/local/ /usr/local/
-
-RUN apk add clang lld pkgconfig
+        musl-dev gcc lld binutils-gold
 
 WORKDIR /root/smoketest
 COPY tests/smoketest/go.mod .
 COPY tests/smoketest/go.sum .
 RUN go mod download
 
-ARG TARGETPLATFORM
-
-# Some dependencies have to installed 
-# for the target platform: https://github.com/tonistiigi/xx#go--cgo
-RUN xx-apk add --no-cache \
-        musl-dev \
-        gcc
-
-
 COPY tests/smoketest/main.go .
 
 ENV CGO_ENABLED=1
@@ -119,11 +51,4 @@ COPY --from=build \
 
 ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
 
-# To do docker run instead, replace the RUN statement with:
-# ENTRYPOINT [ "/root/smoketest/static-test-runner" ]
-
-# The approach below was preferred as it provides a way to 
-# assert the functionality across the supported architectures 
-# without any extra steps.
-
 RUN /root/smoketest/static-test-runner

Makefile

@@ -1,10 +1,41 @@
-IMG ?= hiddeco/golang-with-libgit2
+IMG ?= ghcr.io/fluxcd/golang-with-libgit2
 TAG ?= latest
-STATIC_TEST_TAG := test
 
 PLATFORMS ?= linux/amd64,linux/arm/v7,linux/arm64
 BUILD_ARGS ?=
 
+REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel)
+TARGET_DIR ?= $(REPOSITORY_ROOT)/build/libgit2
+BUILD_ROOT_DIR ?= $(REPOSITORY_ROOT)/build/libgit2-src
+
+LIBGIT2_PATH := $(TARGET_DIR)
+LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
+LIBGIT2_LIB64_PATH := $(LIBGIT2_PATH)/lib64
+LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
+
+export CGO_ENABLED=1
+export LIBRARY_PATH=$(LIBGIT2_LIB_PATH):$(LIBGIT2_LIB64_PATH)
+export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig:$(LIBGIT2_LIB64_PATH)/pkgconfig
+export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include
+
+
+ifeq ($(shell uname -s),Linux)
+	export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2) -static
+else
+	export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2) -Wl,--unresolved-symbols=ignore-in-object-files -Wl,-allow-shlib-undefined -static
+endif
+
+ifeq ($(shell uname -s),Linux)
+ifneq ($(shell uname -m),x86_64)
+	export CC=$(shell uname -m)-linux-musl-gcc
+	export CXX=$(shell uname -m)-linux-musl-g++
+	export AR=$(shell uname -m)-linux-musl-ar
+endif
+endif
+
+GO_STATIC_FLAGS=-tags 'netgo,osusergo,static_build'
+
+
 .PHONY: build
 build:
 	docker buildx build \
@@ -18,6 +49,8 @@ test:
 	docker buildx build \
 		--platform=$(PLATFORMS) \
 		--tag $(IMG):$(TAG) \
+		--build-arg LIBGIT2_IMG=$(IMG) \
+		--build-arg LIBGIT2_TAG=$(TAG) \
 		--file Dockerfile.test \
 		$(BUILD_ARGS) .
 
@@ -33,3 +66,17 @@ builder:
 		--use
 # install qemu emulators
 	docker run -it --rm --privileged tonistiigi/binfmt --install all
+
+$(LIBGIT2):
+ifeq ($(shell uname -s),Darwin)
+	TARGET_DIR=$(TARGET_DIR) BUILD_ROOT_DIR=$(BUILD_ROOT_DIR) \
+		./hack/static.sh all
+else
+	IMG_TAG=$(IMG):$(TAG) ./hack/extract-libraries.sh
+endif
+
+# dev-test is a smoke test for development environment
+# consuming the libraries generated by this project.
+dev-test: $(LIBGIT2)
+	cd tests/smoketest; go vet $(GO_STATIC_FLAGS) ./...
+	cd tests/smoketest; go run $(GO_STATIC_FLAGS) main.go

hack/extract-libraries.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+IMG_TAG="${IMG_TAG:-.}"
+
+function extract(){
+    PLATFORM=$1
+    DIR=$2
+
+    id=$(docker create --platform="${PLATFORM}" "${IMG_TAG}")
+    docker cp "${id}":/usr/local - > output.tar.gz
+    docker rm -v "${id}"
+
+    tar -xf output.tar.gz "local/${DIR}"
+    rm output.tar.gz
+}
+
+function setup() {
+    PLATFORM=$1
+    DIR=$2
+
+    extract "${PLATFORM}" "${DIR}"
+   
+    NEW_DIR="$(/bin/pwd)/build/libgit2"
+    INSTALLED_DIR="/usr/local/${DIR}"
+
+    mkdir -p "./build"
+
+    mv "local/${DIR}/" "libgit2"
+    rm -rf "local"
+    mv "libgit2/" "./build/"
+
+    # Update the prefix paths included in the .pc files.
+    # This will make it easier to update to the location in which they will be used.
+    if [[ $OSTYPE == 'darwin'* ]]; then    
+        # sed has a sight different behaviour in MacOS
+        find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "" "s;${INSTALLED_DIR};${NEW_DIR};g" {}
+    else
+        find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
+    fi
+}
+
+function setup_current() {
+    if [ -d "./build/libgit2" ]; then
+        echo "Skipping libgit2 setup as it already exists"
+        exit 0
+    fi
+
+    DIR="x86_64-alpine-linux-musl"
+    PLATFORM="linux/amd64"
+
+    if [[ "$(uname -m)" == armv7* ]]; then 
+        DIR="armv7-alpine-linux-musleabihf"
+        PLATFORM="linux/arm/v7"
+    elif [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+        DIR="aarch64-alpine-linux-musl"
+        PLATFORM="linux/arm64"
+    fi
+    
+    setup "${PLATFORM}" "${DIR}"
+}
+
+setup_current

hack/static.sh

@@ -14,13 +14,12 @@ TARGET_DIR="${TARGET_DIR:-/usr/local/$(xx-info triple)}"
 BUILD_ROOT_DIR="${BUILD_ROOT_DIR:-/build}"
 SRC_DIR="${BUILD_ROOT_DIR}/src"
 
-
 TARGET_ARCH="$(uname -m)"
 if command -v xx-info; then 
     TARGET_ARCH="$(xx-info march)"
 fi
 
-C_COMPILER="/usr/bin/gcc"
+C_COMPILER="${CC:-/usr/bin/gcc}"
 CMAKE_PARAMS=""
 if command -v xx-clang; then 
     C_COMPILER="/usr/bin/xx-clang"
@@ -40,6 +39,7 @@ function build_libz(){
     download_source "${LIBZ_URL}" "${SRC_DIR}/libz"    
     pushd "${SRC_DIR}/libz"
 
+    # if target architecture is the same as current, no cross compiling is required
     if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then
         ./configure --static --prefix="${TARGET_DIR}"
     else
@@ -56,14 +56,24 @@ function build_openssl(){
     download_source "${OPENSSL_URL}" "${SRC_DIR}/openssl"    
     pushd "${SRC_DIR}/openssl"
 
-    target_name="${TARGET_ARCH}"
-    if [ "${target_name}" = "armv7l" ]; then
+    export OPENSSL_ROOT_DIR="${TARGET_DIR}"
+    export OPENSSL_LIBRARIES="${TARGET_DIR}/lib"
+
+    target_arch=""
+    if [ "${TARGET_ARCH}" = "armv7l" ]; then
         # openssl does not have a specific armv7l
         # using generic32 instead.
-        target_name=generic32
+        target_arch="linux-generic32"
+    elif [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
+        target_arch="linux-aarch64"  
+    elif [ "${TARGET_ARCH}" = "x86_64" ]; then
+        target_arch="linux-x86_64"
+    else
+        echo "Architecture currently not supported: ${TARGET_ARCH}"
+        exit 1
     fi
 
-    ./Configure "linux-${target_name}" threads no-shared zlib -fPIC -DOPENSSL_PIC \
+    ./Configure "${target_arch}" threads no-shared zlib -fPIC -DOPENSSL_PIC \
         --prefix="${TARGET_DIR}" \
         --with-zlib-include="${TARGET_DIR}/include" \
         --with-zlib-lib="${TARGET_DIR}/lib" \
@@ -83,14 +93,23 @@ function build_libssh2(){
     mkdir -p build
     pushd build
 
+    OPENSSL_LIBRARIES="${TARGET_DIR}/lib"
+    if [ "${TARGET_ARCH}" = "x86_64" ]; then
+        OPENSSL_LIBRARIES="${TARGET_DIR}/lib64"
+    fi
+
+
     cmake "${CMAKE_PARAMS}" \
         -DCMAKE_C_COMPILER="${C_COMPILER}" \
         -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \
         -DBUILD_SHARED_LIBS=OFF \
+        -DLINT=OFF \
         -DCMAKE_C_FLAGS=-fPIC \
         -DCRYPTO_BACKEND=OpenSSL \
         -DENABLE_ZLIB_COMPRESSION=ON \
         -DCMAKE_BUILD_TYPE="RelWithDebInfo" \
+        -DOPENSSL_CRYPTO_LIBRARY="${OPENSSL_LIBRARIES}/libcrypto.a" \
+        -DOPENSSL_SSL_LIBRARY="${OPENSSL_LIBRARIES}/libssl.a" \
         ..
 
     cmake --build . --target install
@@ -108,11 +127,19 @@ function build_libgit2(){
 
     pushd build
 
+    SSL_LIBRARY="${TARGET_DIR}/lib/libssl.a"
+    CRYPTO_LIBRARY="${TARGET_DIR}/lib/libcrypto.a"
+    if [ "${TARGET_ARCH}" = "x86_64" ]; then
+        SSL_LIBRARY="${TARGET_DIR}/lib64/libssl.a"
+        CRYPTO_LIBRARY="${TARGET_DIR}/lib64/libcrypto.a"
+    fi
+
     cmake "${CMAKE_PARAMS}" \
         -DCMAKE_C_COMPILER="${C_COMPILER}" \
         -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \
         -DTHREADSAFE:BOOL=ON \
-        -DBUILD_CLAR:BOOL:BOOL=OFF \
+        -DBUILD_CLAR:BOOL=OFF \
+        -DBUILD_TESTS:BOOL=OFF \
         -DBUILD_SHARED_LIBS=OFF \
         -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \
         -DCMAKE_C_FLAGS=-fPIC \
@@ -122,6 +149,9 @@ function build_libgit2(){
         -DUSE_BUNDLED_ZLIB:BOOL=ON \
         -DUSE_HTTPS:STRING=OpenSSL \
         -DREGEX_BACKEND:STRING=builtin \
+        -DOPENSSL_SSL_LIBRARY="${SSL_LIBRARY}" \
+        -DOPENSSL_CRYPTO_LIBRARY="${CRYPTO_LIBRARY}" \
+        -DZLIB_LIBRARY="${TARGET_DIR}/lib/libz.a" \
         -DCMAKE_INCLUDE_PATH="${TARGET_DIR}/include" \
         -DCMAKE_LIBRARY_PATH="${TARGET_DIR}/lib" \
         -DCMAKE_BUILD_TYPE="RelWithDebInfo" \