Work with the Host Filesystem

Introduction

This guide explains how to work with the host filesystem using the Dagger SDKs. You will learn how to:

• Set the working directory on the host
• List host directory entries with include/exclude filters
• Mount a host directory in a container
• Export a directory from a container to the host

Requirements

This guide assumes that:

• You have a Go, Python or Node.js development environment. If not, install Go, Python or Node.js.
• You have a Dagger SDK installed for one of the above languages. If not, follow the installation instructions for the Dagger Go, Python or Node.js SDK.
• You have the Dagger CLI installed in your development environment. If not, install the Dagger CLI.
• You have Docker installed and running on the host system. If not, install Docker.

List directory contents

The easiest way to set the working directory for the Dagger CI pipeline is at the time of client instantiation, as a client configuration option. By default, Dagger uses the current directory on the host as the working directory.

The following example shows how to list the contents of the working directory:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"dagger.io/dagger"
)

func main() {
	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	entries, err := client.Host().Directory(".").Entries(ctx)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println(entries)
}

The Host type provides information about the host's execution environment. Its Directory() method accepts a path and returns a reference to the corresponding host directory as a Directory struct. Entries in the directory can be obtained via the Directory.Entries() function.

Node.js

import { connect, Client } from "@dagger.io/dagger"

connect(
  async (client: Client) => {
    const entries = await client.host().directory(".").entries()
    console.log(entries)
  },
  { LogOutput: process.stderr },
)

The host type provides information about the host's execution environment. Its directory() method accepts a path and returns a reference to the corresponding host directory as a Directory object. Entries in the directory can be obtained via the directory.entries() function.

Python

import sys

import anyio

import dagger


async def main():
    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
        entries = await client.host().directory(".").entries()

    print(entries)


anyio.run(main)

The host type provides information about the host's execution environment. Its directory() method accepts a path and returns a reference to the corresponding host directory as a Directory object. Entries in the directory can be obtained via the directory.entries() function.

List directory contents with filters

It's possible to restrict a Directory to a subset of directory entries, by specifying a list of filename patterns to include or exclude.

The following example shows how to obtain a reference to the host working directory containing only *.rar files:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"dagger.io/dagger"
)

func main() {
	os.WriteFile("foo.txt", []byte("1"), 0600)
	os.WriteFile("bar.txt", []byte("2"), 0600)
	os.WriteFile("baz.rar", []byte("3"), 0600)

	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	entries, err := client.Host().Directory(".", dagger.HostDirectoryOpts{
		Include: []string{"*.rar"},
	}).Entries(ctx)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println(entries)
}

Node.js

import { connect, Client } from "@dagger.io/dagger"
import * as fs from "fs"

const files = ["foo.txt", "bar.txt", "baz.rar"]
let count = 1

for (const file of files) {
  fs.writeFileSync(file, count.toString())
  count = count + 1
}

connect(
  async (client: Client) => {
    const entries = await client
      .host()
      .directory(".", { include: ["*.rar"] })
      .entries()
    console.log(entries)
  },
  { LogOutput: process.stderr },
)

Python

import sys

import anyio

import dagger


async def main():
    for i, file in enumerate(["foo.txt", "bar.txt", "baz.rar"]):
        await (anyio.Path(".") / file).write_text(str(i + 1))

    cfg = dagger.Config(log_output=sys.stderr)

    async with dagger.Connection(cfg) as client:
        entries = await client.host().directory(".", include=["*.rar"]).entries()

    print(entries)


anyio.run(main)

The following example shows how to obtain a reference to the host working directory containing all files except *.txt files:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"dagger.io/dagger"
)

func main() {
	os.WriteFile("foo.txt", []byte("1"), 0600)
	os.WriteFile("bar.txt", []byte("2"), 0600)
	os.WriteFile("baz.rar", []byte("3"), 0600)

	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	entries, err := client.Host().Directory(".", dagger.HostDirectoryOpts{
		Exclude: []string{"*.txt"},
	}).Entries(ctx)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println(entries)
}

Node.js

import { connect, Client } from "@dagger.io/dagger"
import * as fs from "fs"

const files = ["foo.txt", "bar.txt", "baz.rar"]
let count = 1

for (const file of files) {
  fs.writeFileSync(file, count.toString())
  count = count + 1
}

connect(
  async (client: Client) => {
    const entries = await client
      .host()
      .directory(".", { exclude: ["*.txt"] })
      .entries()
    console.log(entries)
  },
  { LogOutput: process.stderr },
)

Python

import sys

import anyio

import dagger


async def main():
    for i, file in enumerate(["foo.txt", "bar.txt", "baz.rar"]):
        await (anyio.Path(".") / file).write_text(str(i + 1))

    cfg = dagger.Config(log_output=sys.stderr)

    async with dagger.Connection(cfg) as client:
        entries = await client.host().directory(".", exclude=["*.txt"]).entries()

    print(entries)


anyio.run(main)

The exclusion pattern overrides the inclusion pattern, but not vice-versa. The following example demonstrates by obtaining a reference to the host working directory containing all files except *.rar files:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"dagger.io/dagger"
)

func main() {
	os.WriteFile("foo.txt", []byte("1"), 0600)
	os.WriteFile("bar.txt", []byte("2"), 0600)
	os.WriteFile("baz.rar", []byte("3"), 0600)

	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	entries, err := client.Host().Directory(".", dagger.HostDirectoryOpts{
		Include: []string{"*.*"},
		Exclude: []string{"*.rar"},
	}).Entries(ctx)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println(entries)
}

Node.js

import { connect, Client } from "@dagger.io/dagger"
import * as fs from "fs"

const files = ["foo.txt", "bar.txt", "baz.rar"]
let count = 1

for (const file of files) {
  fs.writeFileSync(file, count.toString())
  count = count + 1
}

connect(
  async (client: Client) => {
    const entries = await client
      .host()
      .directory(".", { include: ["*.*"], exclude: ["*.rar"] })
      .entries()
    console.log(entries)
  },
  { LogOutput: process.stderr },
)

Python

import sys

import anyio

import dagger


async def main():
    for i, file in enumerate(["foo.txt", "bar.txt", "baz.rar"]):
        await (anyio.Path(".") / file).write_text(str(i + 1))

    cfg = dagger.Config(log_output=sys.stderr)

    async with dagger.Connection(cfg) as client:
        entries = (
            await client.host()
            .directory(".", exclude=["*.rar"], include=["*.*"])
            .entries()
        )

    print(entries)


anyio.run(main)

The exclusion pattern overrides the inclusion pattern, but not vice-versa. The following example demonstrates by obtaining a reference to the host working directory containing all .rar and .txt files except .out files using glob patterns:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"dagger.io/dagger"
)

func main() {
	workdir, _ := os.Getwd()
	folder := workdir + string(os.PathSeparator)

	for _, subdir := range []string{"foo", "bar", "baz"} {
		folder = filepath.Join(folder, subdir)
		os.Mkdir(folder, 0700)

		for _, file := range []string{".txt", ".rar", ".out"} {
			os.WriteFile(filepath.Join(folder, subdir+file), []byte(subdir), 0600)
		}
	}

	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	daggerdir := client.Host().Directory(".", dagger.HostDirectoryOpts{
		Include: []string{"**/*.rar", "**/*.txt"},
		Exclude: []string{"**.out"},
	})

	folder = "." + string(os.PathSeparator)
	for _, d := range []string{"foo", "bar", "baz"} {
		folder = filepath.Join(folder, d)
		entries, err := daggerdir.Entries(ctx, dagger.DirectoryEntriesOpts{Path: folder})
		if err != nil {
			log.Println(err)
			return
		}
		fmt.Printf("In %s: %v\n", folder, entries)
	}
}

Node.js

import { connect, Client } from "@dagger.io/dagger"
import * as fs from "fs"
import * as path from "path"

const workdir = process.cwd()
let folder = workdir + path.sep

for (const subdir of ["foo", "bar", "baz"]) {
  folder = path.join(folder, subdir)
  fs.mkdirSync(folder)

  for (const file of [".txt", ".out", ".rar"]) {
    fs.writeFileSync(path.join(folder, subdir + file), subdir)
  }
}

connect(
  async (client: Client) => {
    const daggerdir = await client.host().directory(workdir, {
      include: ["**/*.rar", "**/*.txt"],
      exclude: ["**.out"],
    })

    folder = "." + path.sep
    for (const dir of ["foo", "bar", "baz"]) {
      folder = path.join(folder, dir)
      const entries = await daggerdir.entries({ path: folder })
      console.log(entries)
    }
  },
  { LogOutput: process.stderr },
)

Python

import os
import sys
from pathlib import Path

import anyio

import dagger


async def main(hostdir: str):
    folder = Path(hostdir)
    for subdir in ["foo", "bar", "baz"]:
        folder.joinpath(Path(subdir)).mkdir()
        for file in [".txt", ".out", ".rar"]:
            folder.joinpath(Path(subdir), str(subdir + file)).write_text(str(subdir))
        folder = folder / subdir

    cfg = dagger.Config(log_output=sys.stderr)

    async with dagger.Connection(cfg) as client:
        daggerdirectory = await client.host().directory(
            ".", include=["**/*.rar", "**/*.txt"], exclude=["**.out"]
        )

        folder = "." + os.sep
        for _, subdir in enumerate(["foo", "bar", "baz"]):
            folder = folder = Path(folder) / subdir
            entries = await daggerdirectory.entries(path=str(folder))
            print("In", subdir, ":", entries)


hostdir = Path.cwd()
anyio.run(main, hostdir)

Export a directory from a container to the host

A directory can be exported to a different path. The destination path is supplied to the method as an argument.

The following example creates a file in a container's /tmp directory and then exports the contents of that directory to the host's temporary directory:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"dagger.io/dagger"
)

func main() {
	hostdir := os.TempDir()

	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	_, err = client.Container().From("alpine:latest").
		WithWorkdir("/tmp").
		WithExec([]string{"wget", "https://dagger.io"}).
		Directory(".").
		Export(ctx, hostdir)
	if err != nil {
		log.Println(err)
		return
	}
	contents, err := os.ReadFile(filepath.Join(hostdir, "index.html"))
	if err != nil {
		log.Println(err)
		return
	}
	fmt.Println(string(contents))
}

Node.js

import { connect, Client } from "@dagger.io/dagger"
import * as fs from "fs"
import * as os from "os"
import * as path from "path"

const hostdir = os.tmpdir()

connect(
  async (client: Client) => {
    await client
      .container()
      .from("alpine:latest")
      .withWorkdir("/tmp")
      .withExec(["wget", "https://dagger.io"])
      .directory(".")
      .export(hostdir)

    const contents = fs.readFileSync(path.join(hostdir, "index.html"))

    console.log(contents.toString())
  },
  { LogOutput: process.stderr },
)

Python

import sys
import tempfile

import anyio

import dagger


async def main(hostdir: str):
    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
        await (
            client.container()
            .from_("alpine:latest")
            .with_workdir("/tmp")
            .with_exec(["wget", "https://dagger.io"])
            .directory(".")
            .export(hostdir)
        )

    contents = await anyio.Path(hostdir, "index.html").read_text()

    print(contents)


with tempfile.TemporaryDirectory() as hostdir:
    anyio.run(main, hostdir)

Write a host directory to a container

A common operation when working with containers is to write a host directory to a path in the container and then perform operations on it. It is necessary to provide the filesystem location in the container and the directory to be written as method arguments.

The following example shows how to write a host directory to a container at the /host container path and then read the contents of the directory:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"dagger.io/dagger"
)

func main() {
	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	contents, err := client.Container().
		From("alpine:latest").
		WithDirectory("/host", client.Host().Directory(".")).
		WithExec([]string{"ls", "/host"}).
		Stdout(ctx)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println(contents)
}

Node.js

import { connect, Client } from "@dagger.io/dagger"

connect(
  async (client: Client) => {
    const contents = await client
      .container()
      .from("alpine:latest")
      .withDirectory("/host", client.host().directory("."))
      .withExec(["ls", "/host"])
      .stdout()

    console.log(contents)
  },
  { LogOutput: process.stderr },
)

Python

import sys

import anyio

import dagger


async def main():
    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
        out = await (
            client.container()
            .from_("alpine:latest")
            .with_directory("/host", client.host().directory("."))
            .with_exec(["ls", "/host"])
            .stdout()
        )

    print(out)


anyio.run(main)

note

Modifications made to a host directory written to a container filesystem path do not appear on the host. Data flows only one way between Dagger operations, because they are connected in a DAG. To write modifications back to the host directory, you must explicitly export the directory back to the host filesystem.

The following example shows how to transfer a host directory to a container at the /host container path, write a file to it, and then export the modified directory back to the host:

Go

package main

import (
	"context"
	"fmt"
	"log"
	"os"

	"dagger.io/dagger"
)

func main() {
	ctx := context.Background()

	client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stderr))
	if err != nil {
		log.Println(err)
		return
	}
	defer client.Close()

	contents, err := client.Container().
		From("alpine:latest").
		WithDirectory("/host", client.Host().Directory("/tmp/sandbox")).
		WithExec([]string{"/bin/sh", "-c", `echo foo > /host/bar`}).
		Directory("/host").
		Export(ctx, "/tmp/sandbox")
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println(contents)
}

Node.js

import { connect, Client } from "@dagger.io/dagger"

connect(
  async (client: Client) => {
    const contents = await client
      .container()
      .from("alpine:latest")
      .withDirectory("/host", client.host().directory("/tmp/sandbox"))
      .withExec(["/bin/sh", "-c", `echo foo > /host/bar`])
      .directory("/host")
      .export("/tmp/sandbox")

    console.log(contents)
  },
  { LogOutput: process.stderr },
)

Python

import sys

import anyio

import dagger


async def main():
    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
        out = await (
            client.container()
            .from_("alpine:latest")
            .with_directory("/host", client.host().directory("/tmp/sandbox"))
            .with_exec(["/bin/sh", "-c", "`echo foo > /host/bar`"])
            .directory("/host")
            .export("/tmp/sandbox")
        )

    print(out)


anyio.run(main)

Important notes

Using the host filesystem in your Dagger pipeline is convenient, but there are some important considerations to keep in mind:

• With the exception of mounted cache volumes, if a file or directory mounted from the host changes even slightly (including minor changes such as a timestamp change with the file contents left unmodified), then the Dagger pipeline operations cache will be invalidated. An extremely common source of invalidations occurs when loading the .git directory from the host filesystem, as that directory will change frequently, including when there have been no actual changes to any source code.

  tip

  To maximize cache re-use, it's important to use the include/exclude options for local directories to only include the files/directories needed for the pipeline. Excluding the .git directory is highly advisable unless there's a strong need to be able to perform Git operations on top of the loaded directory inside Dagger.

• The host directory is synchronized into the Dagger Engine similar to rsync or scp; it's not a "bind mount". This means that any change you make to the loaded directory in your Dagger pipeline will not result in a change to the directory on the host.

  warning

  If you want the changes made to a loaded local directory inside a Dagger pipeline to be reflected on the host, it needs to be explictly exported to the host. However, this should be approached with caution, since any overlap in the files being exported with the files on the host will result in the host files being overwritten.

• Synchronization of a local directory happens once per Dagger client instance (in user-facing terms, once per dagger.Connect call in the Dagger SDKs). This means that if you load the local directory, then make changes to it on the host, those changes will not be reloaded within the context of a single Dagger client. Furthermore, due to lazy executions, the loading happens the first time the directory is used in a non-lazy operation.

  tip

  It's safest to not modify a loaded host directory at all while a Dagger client is running, as otherwise it is hard to predict what will be loaded.

Conclusion

This guide introduced you to the functions available in the Dagger SDKs to work with the host filesystem. It provided explanations and code samples demonstrating how to set the host working directory, read directory contents (with and without pathname filters), mount a host directory in a container and export a directory from a container to the host.

Use the Go, Node.js and Python SDK References to learn more about Dagger.