Merge pull request #32248 from awakesecurity/parnell/fetchdocker

Support fetching docker images from V2 registries
This commit is contained in:
Franz Pletz 2018-03-04 17:10:27 +00:00 committed by GitHub
commit 0f78afdf25
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 317 additions and 0 deletions

View File

@ -274,6 +274,7 @@ in rec {
tests.hibernate = callTest tests/hibernate.nix {};
tests.home-assistant = callTest tests/home-assistant.nix { };
tests.hound = callTest tests/hound.nix {};
tests.hocker-fetchdocker = callTest tests/hocker-fetchdocker {};
tests.i3wm = callTest tests/i3wm.nix {};
tests.initrd-network-ssh = callTest tests/initrd-network-ssh {};
tests.installer = callSubTests tests/installer.nix {};

View File

@ -0,0 +1,15 @@
import ../make-test.nix ({ pkgs, ...} : {
name = "test-hocker-fetchdocker";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ ixmatus ];
};
machine = import ./machine.nix;
testScript = ''
startAll;
$machine->waitForUnit("sockets.target");
$machine->waitUntilSucceeds("docker run registry-1.docker.io/v2/library/hello-world:latest");
'';
})

View File

@ -0,0 +1,19 @@
{ fetchDockerConfig, fetchDockerLayer, fetchdocker }:
fetchdocker rec {
name = "hello-world";
registry = "https://registry-1.docker.io/v2/";
repository = "library";
imageName = "hello-world";
tag = "latest";
imageConfig = fetchDockerConfig {
inherit tag registry repository imageName;
sha256 = "1ivbd23hyindkahzfw4kahgzi6ibzz2ablmgsz6340vc6qr1gagj";
};
imageLayers = let
layer0 = fetchDockerLayer {
inherit registry repository imageName;
layerDigest = "ca4f61b1923c10e9eb81228bd46bee1dfba02b9c7dac1844527a734752688ede";
sha256 = "1plfd194fwvsa921ib3xkhms1yqxxrmx92r2h7myj41wjaqn2kya";
};
in [ layer0 ];
}

View File

@ -0,0 +1,26 @@
{ config, pkgs, ... }:
{ nixpkgs.config.packageOverrides = pkgs': {
hello-world-container = pkgs'.callPackage ./hello-world-container.nix { };
};
virtualisation.docker = {
enable = true;
package = pkgs.docker;
};
systemd.services.docker-load-fetchdocker-image = {
description = "Docker load hello-world-container";
wantedBy = [ "multi-user.target" ];
wants = [ "docker.service" "local-fs.target" ];
after = [ "docker.service" "local-fs.target" ];
script = ''
${pkgs.hello-world-container}/compositeImage.sh | ${pkgs.docker}/bin/docker load
'';
serviceConfig = {
Type = "oneshot";
};
};
}

View File

@ -0,0 +1,38 @@
# We provide three paths to get the credentials into the builder's
# environment:
#
# 1. Via impureEnvVars. This method is difficult for multi-user Nix
# installations (but works very well for single-user Nix
# installations!) because it requires setting the environment
# variables on the nix-daemon which is either complicated or unsafe
# (i.e: configuring via Nix means the secrets will be persisted
# into the store)
#
# 2. If the DOCKER_CREDENTIALS key with a path to a credentials file
# is added to the NIX_PATH (usually via the '-I ' argument to most
# Nix tools) then an attempt will be made to read credentials from
# it. The semantics are simple, the file should contain two lines
# for the username and password based authentication:
#
# $ cat ./credentials-file.txt
# DOCKER_USER=myusername
# DOCKER_PASS=mypassword
#
# ... and a single line for the token based authentication:
#
# $ cat ./credentials-file.txt
# DOCKER_TOKEN=mytoken
#
# 3. A credential file at /etc/nix-docker-credentials.txt with the
# same format as the file described in #2 can also be used to
# communicate credentials to the builder. This is necessary for
# situations (like Hydra) where you cannot customize the NIX_PATH
# given to the nix-build invocation to provide it with the
# DOCKER_CREDENTIALS path
let
pathParts =
(builtins.filter
({path, prefix}: "DOCKER_CREDENTIALS" == prefix)
builtins.nixPath);
in
if (pathParts != []) then (builtins.head pathParts).path else ""

View File

@ -0,0 +1,61 @@
{ stdenv, lib, coreutils, bash, gnutar, jq, writeText }:
let
stripScheme =
builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
stripNixStore =
s: lib.removePrefix "/nix/store/" s;
in
{ name
, registry ? "https://registry-1.docker.io/v2/"
, repository ? "library"
, imageName
, tag
, imageLayers
, imageConfig
, image ? "${stripScheme registry}/${repository}/${imageName}:${tag}"
}:
# Make sure there are *no* slashes in the repository or container
# names since we use these to make the output derivation name for the
# nix-store path.
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
let
# Abuse `builtins.toPath` to collapse possible double slashes
repoTag0 = builtins.toString (builtins.toPath "/${stripScheme registry}/${repository}/${imageName}");
repoTag1 = lib.removePrefix "/" repoTag0;
layers = builtins.map stripNixStore imageLayers;
manifest =
writeText "manifest.json" (builtins.toJSON [
{ Config = stripNixStore imageConfig;
Layers = layers;
RepoTags = [ "${repoTag1}:${tag}" ];
}]);
repositories =
writeText "repositories" (builtins.toJSON {
"${repoTag1}" = {
"${tag}" = lib.last layers;
};
});
imageFileStorePaths =
writeText "imageFileStorePaths.txt"
(lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [imageConfig]));
in
stdenv.mkDerivation {
builder = ./fetchdocker-builder.sh;
buildInputs = [ coreutils ];
preferLocalBuild = true;
inherit name imageName repository tag;
inherit bash gnutar manifest repositories;
inherit imageFileStorePaths;
passthru = {
inherit image;
};
}

View File

@ -0,0 +1,13 @@
pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
let
generic-fetcher =
import ./generic-fetcher.nix pkgargs;
in
args@{ repository ? "library", imageName, tag, ... }:
generic-fetcher ({
fetcher = "hocker-config";
name = "${repository}_${imageName}_${tag}-config.json";
tag = "unused";
} // args)

View File

@ -0,0 +1,13 @@
pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
let
generic-fetcher =
import ./generic-fetcher.nix pkgargs;
in
args@{ layerDigest, ... }:
generic-fetcher ({
fetcher = "hocker-layer";
name = "docker-layer-${layerDigest}.tar.gz";
tag = "unused";
} // args)

View File

@ -0,0 +1,28 @@
source "${stdenv}/setup"
header "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}"
mkdir -p "${out}"
cat <<EOF > "${out}/compositeImage.sh"
#! ${bash}/bin/bash
#
# Create a tar archive of a docker image's layers, docker image config
# json, manifest.json, and repositories json; this streams directly to
# stdout and is intended to be used in concert with docker load, i.e:
#
# ${out}/compositeImage.sh | docker load
# The first character follow the 's' command for sed becomes the
# delimiter sed will use; this makes the transformation regex easy to
# read. We feed tar a file listing the files we want in the archive,
# because the paths are absolute and docker load wants them flattened in
# the archive, we need to transform all of the paths going in by
# stripping everything *including* the last solidus so that we end up
# with the basename of the path.
${gnutar}/bin/tar \
--transform='s=.*/==' \
--transform="s=.*-manifest.json=manifest.json=" \
--transform="s=.*-repositories=repositories=" \
-c "${manifest}" "${repositories}" -T "${imageFileStorePaths}"
EOF
chmod +x "${out}/compositeImage.sh"
stopNest

View File

@ -0,0 +1,97 @@
{ stdenv, lib, haskellPackages, writeText, gawk }:
let
awk = "${gawk}/bin/awk";
dockerCredentialsFile = import ./credentials.nix;
stripScheme =
builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
in
{ fetcher
, name
, registry ? "https://registry-1.docker.io/v2/"
, repository ? "library"
, imageName
, sha256
, tag ? ""
, layerDigest ? ""
}:
# There must be no slashes in the repository or container names since
# we use these to make the output derivation name for the nix store
# path
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
# Only allow hocker-config and hocker-layer as fetchers for now
assert (builtins.elem fetcher ["hocker-config" "hocker-layer"]);
# If layerDigest is non-empty then it must not have a 'sha256:' prefix!
assert
(if layerDigest != ""
then !lib.hasPrefix "sha256:" layerDigest
else true);
let
layerDigestFlag =
lib.optionalString (layerDigest != "") "--layer ${layerDigest}";
in
stdenv.mkDerivation {
inherit name;
builder = writeText "${fetcher}-builder.sh" ''
source "$stdenv/setup"
header "${fetcher} exporting to $out"
declare -A creds
# This is a hack for Hydra since we have no way of adding values
# to the NIX_PATH for Hydra jobsets!!
staticCredentialsFile="/etc/nix-docker-credentials.txt"
if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then
echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile"
dockerCredentialsFile=$staticCredentialsFile
fi
if [ -f "$dockerCredentialsFile" ]; then
header "using credentials from $dockerCredentialsFile"
CREDSFILE=$(cat "$dockerCredentialsFile")
creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1)
# Prefer DOCKER_TOKEN over the username and password
# authentication method
if [ -z "''${creds[token]}" ]; then
creds[user]=$(${awk} -F'=' '/DOCKER_USER/ {print $2}' <<< "$CREDSFILE" | head -n1)
creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/ {print $2}' <<< "$CREDSFILE" | head -n1)
fi
fi
# These variables will be filled in first by the impureEnvVars, if
# those variables are empty then they will default to the
# credentials that may have been read in from the 'DOCKER_CREDENTIALS'
DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}"
DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}"
DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}"
${fetcher} --out="$out" \
''${registry:+--registry "$registry"} \
''${DOCKER_USER:+--username "$DOCKER_USER"} \
''${DOCKER_PASS:+--password "$DOCKER_PASS"} \
''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \
${layerDigestFlag} \
"${repository}/${imageName}" \
"${tag}"
stopNest
'';
buildInputs = [ haskellPackages.hocker ];
outputHashAlgo = "sha256";
outputHashMode = "flat";
outputHash = sha256;
preferLocalBuild = true;
impureEnvVars = [ "DOCKER_USER" "DOCKER_PASS" "DOCKER_TOKEN" ];
inherit registry dockerCredentialsFile;
}

View File

@ -160,6 +160,12 @@ with pkgs;
fetchdarcs = callPackage ../build-support/fetchdarcs { };
fetchdocker = callPackage ../build-support/fetchdocker { };
fetchDockerConfig = callPackage ../build-support/fetchdocker/fetchDockerConfig.nix { };
fetchDockerLayer = callPackage ../build-support/fetchdocker/fetchDockerLayer.nix { };
fetchfossil = callPackage ../build-support/fetchfossil { };
fetchgit = callPackage ../build-support/fetchgit {