feat: add tar and gzip package modules
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
parent
f0a17f15d3
commit
c118822469
|
@ -1,44 +1,44 @@
|
|||
locals {
|
||||
linux_archs = toset(["amd64", "arm64"])
|
||||
linux_archs = toset(["amd64", "arm64"])
|
||||
macos_archs = toset(["arm64"])
|
||||
}
|
||||
|
||||
build "go_build" "linux" {
|
||||
for_each = local.linux_archs
|
||||
for_each = local.linux_archs
|
||||
|
||||
binary = "buildr_linux_${each.key}"
|
||||
main = "."
|
||||
goos = "linux"
|
||||
goarch = each.key
|
||||
binary = "buildr_linux_${each.key}"
|
||||
main = "."
|
||||
goos = "linux"
|
||||
goarch = each.key
|
||||
|
||||
flags = [
|
||||
"-v",
|
||||
"-trimpath",
|
||||
"-a",
|
||||
"-installsuffix=cgo"
|
||||
]
|
||||
flags = [
|
||||
"-v",
|
||||
"-trimpath",
|
||||
"-a",
|
||||
"-installsuffix=cgo"
|
||||
]
|
||||
|
||||
ldflags = [
|
||||
"-w -s",
|
||||
"-X 'code.icb4dc0.de/buildr/buildr/cmd.CurrentVersion=${vcs.tag == "" ? "v0.1.0" : vcs.tag}'"
|
||||
]
|
||||
ldflags = [
|
||||
"-w -s",
|
||||
"-X 'code.icb4dc0.de/buildr/buildr/cmd.CurrentVersion=${vcs.tag == "" ? "v0.1.0" : vcs.tag}'"
|
||||
]
|
||||
|
||||
environment = {
|
||||
CGO_ENABLED = "0"
|
||||
}
|
||||
environment = {
|
||||
CGO_ENABLED = "0"
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
tasks.go_generate.id
|
||||
]
|
||||
depends_on = [
|
||||
tasks.go_generate.id
|
||||
]
|
||||
}
|
||||
|
||||
build "go_build" "darwin" {
|
||||
for_each = local.macos_archs
|
||||
|
||||
binary = "buildr_darwin_${each.key}"
|
||||
main = "."
|
||||
goos = "darwin"
|
||||
goarch = each.key
|
||||
binary = "buildr_darwin_${each.key}"
|
||||
main = "."
|
||||
goos = "darwin"
|
||||
goarch = each.key
|
||||
|
||||
flags = [
|
||||
"-v",
|
||||
|
|
|
@ -1,10 +1,29 @@
|
|||
package "zip_archive" "proto_api" {
|
||||
archive_name = "buildr_linux_amd64.zip"
|
||||
wrap_in_directory = packages.proto_api.name
|
||||
package "gzip_file" "buildr_linux" {
|
||||
for_each = local.linux_archs
|
||||
|
||||
input_mapping = {
|
||||
"api" = "."
|
||||
}
|
||||
src = "buildr_linux_${each.key}"
|
||||
|
||||
input_mapping = {
|
||||
"${builds.linux[each.key].out_dir}" = "."
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
builds.linux[each.key].id
|
||||
]
|
||||
}
|
||||
|
||||
package "gzip_file" "buildr_darwin" {
|
||||
for_each = local.macos_archs
|
||||
|
||||
src = "buildr_darwin_${each.key}"
|
||||
|
||||
input_mapping = {
|
||||
"${builds.darwin[each.key].out_dir}" = "."
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
builds.darwin[each.key].id
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -29,8 +29,7 @@ task "script" "go_test" {
|
|||
}
|
||||
|
||||
depends_on = [
|
||||
tasks.go_generate.id,
|
||||
tools.gotestsum.id
|
||||
tasks.go_generate.id
|
||||
]
|
||||
|
||||
input_mapping = {
|
||||
|
|
|
@ -44,8 +44,7 @@ steps:
|
|||
- apk add -U --no-cache fuse
|
||||
- go install -trimpath -ldflags="-s -w" .
|
||||
- buildr plugins update
|
||||
- buildr --execution.log-to-stderr task go_generate
|
||||
- buildr --execution.log-to-stderr task go_fmt
|
||||
- buildr --execution.log-to-stderr task go_test
|
||||
- buildr --execution.log-to-stderr task golangci_lint
|
||||
volumes:
|
||||
- name: go-cache
|
||||
|
|
16
modules/packaging/archive/common.go
Normal file
16
modules/packaging/archive/common.go
Normal file
|
@ -0,0 +1,16 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
var ErrEmptyArchiveName = errors.New("archive name may not be empty")
|
||||
|
||||
type noOpWritecloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (*noOpWritecloser) Close() error {
|
||||
return nil
|
||||
}
|
76
modules/packaging/archive/gzip.go
Normal file
76
modules/packaging/archive/gzip.go
Normal file
|
@ -0,0 +1,76 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/internal/ioutils"
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
)
|
||||
|
||||
var (
|
||||
_ modules.Module = (*GzipFileCompression)(nil)
|
||||
_ modules.Helper = (*GzipFileCompression)(nil)
|
||||
)
|
||||
|
||||
type GzipFileCompression struct {
|
||||
Source string `hcl:"src"`
|
||||
OutFile string `hcl:"file_name,optional"`
|
||||
}
|
||||
|
||||
func (GzipFileCompression) Category() modules.Category {
|
||||
return modules.CategoryPackage
|
||||
}
|
||||
|
||||
func (GzipFileCompression) Type() string {
|
||||
return "gzip_file"
|
||||
}
|
||||
|
||||
func (b GzipFileCompression) Execute(ctx modules.ExecutionContext) (err error) {
|
||||
if b.Source == "" {
|
||||
return fmt.Errorf("source file path may not be empty")
|
||||
}
|
||||
|
||||
if !filepath.IsAbs(b.Source) {
|
||||
b.Source = filepath.Join(ctx.WorkingDir(), b.Source)
|
||||
}
|
||||
|
||||
source, err := os.Open(b.Source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open source file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.Join(err, source.Close())
|
||||
}()
|
||||
|
||||
if b.OutFile == "" {
|
||||
b.OutFile = filepath.Join(ctx.OutDir(), filepath.Base(b.Source)+".gz")
|
||||
} else if !filepath.IsAbs(b.OutFile) {
|
||||
b.OutFile = filepath.Join(ctx.OutDir(), b.OutFile)
|
||||
}
|
||||
|
||||
outFile, err := os.Create(b.OutFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create output file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.Join(err, outFile.Close())
|
||||
}()
|
||||
|
||||
writer, err := gzip.NewWriterLevel(outFile, gzip.DefaultCompression)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create gzip writer: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
err = errors.Join(err, writer.Close())
|
||||
}()
|
||||
|
||||
_, err = ioutils.CopyWithPooledBuffer(writer, source)
|
||||
|
||||
return err
|
||||
}
|
28
modules/packaging/archive/gzip_help.go
Normal file
28
modules/packaging/archive/gzip_help.go
Normal file
|
@ -0,0 +1,28 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
)
|
||||
|
||||
func (b GzipFileCompression) Help(context.Context) (help modules.Help, err error) {
|
||||
help = modules.Help{
|
||||
Name: b.Type(),
|
||||
Description: "Compress a file using gzip",
|
||||
Examples: []modules.Example{
|
||||
{
|
||||
Name: "Basic example",
|
||||
Description: "Create a gzipped file without specifying an output file.",
|
||||
Spec: &modules.Metadata[GzipFileCompression]{
|
||||
Module: GzipFileCompression{
|
||||
Source: "buildr_linux_amd64",
|
||||
},
|
||||
ModuleName: "linux_amd64_gzip",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return help, nil
|
||||
}
|
93
modules/packaging/archive/gzip_test.go
Normal file
93
modules/packaging/archive/gzip_test.go
Normal file
|
@ -0,0 +1,93 @@
|
|||
package archive_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
"code.icb4dc0.de/buildr/buildr/modules/packaging/archive"
|
||||
"code.icb4dc0.de/buildr/buildr/modules/state"
|
||||
)
|
||||
|
||||
func TestGzipFileCompression_SampleWithoutOutFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := createSampleFile(t, 500*1024)
|
||||
|
||||
gzip := archive.GzipFileCompression{Source: f}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
outDir := t.TempDir()
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
OutputDirectory: outDir,
|
||||
StateStore: new(state.InMemoryStore),
|
||||
}
|
||||
|
||||
if !assert.NoError(t, gzip.Execute(execCtx), "failed to execute gzip module") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
files, err := filepath.Glob(filepath.Join(outDir, "*.gz"))
|
||||
if !assert.NoError(t, err, "failed to glob files") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
assert.Len(t, files, 1)
|
||||
}
|
||||
|
||||
func TestGzipFileCompression_SampleWithOutFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := createSampleFile(t, 500*1024)
|
||||
|
||||
outFile := filepath.Join(t.TempDir(), "out.gz")
|
||||
|
||||
gzip := archive.GzipFileCompression{Source: f, OutFile: outFile}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
StateStore: new(state.InMemoryStore),
|
||||
}
|
||||
|
||||
if !assert.NoError(t, gzip.Execute(execCtx), "failed to execute gzip module") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
assert.FileExists(t, outFile)
|
||||
}
|
||||
|
||||
func TestGzipFileCompression_SampleWithRelativeOutFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := createSampleFile(t, 500*1024)
|
||||
|
||||
outDir := t.TempDir()
|
||||
|
||||
gzip := archive.GzipFileCompression{Source: f, OutFile: "out.gz"}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
StateStore: new(state.InMemoryStore),
|
||||
OutputDirectory: outDir,
|
||||
}
|
||||
|
||||
if !assert.NoError(t, gzip.Execute(execCtx), "failed to execute gzip module") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
assert.FileExists(t, filepath.Join(outDir, "out.gz"))
|
||||
}
|
131
modules/packaging/archive/helpers_test.go
Normal file
131
modules/packaging/archive/helpers_test.go
Normal file
|
@ -0,0 +1,131 @@
|
|||
package archive_test
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zip"
|
||||
"github.com/klauspost/pgzip"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var replacer = strings.NewReplacer("_", "", "-", "", "/", "")
|
||||
|
||||
func createSampleFileAt(tb testing.TB, dir string, size uint64) string {
|
||||
tb.Helper()
|
||||
|
||||
//nolint:gosec // only used for random file creation
|
||||
random := rand.New(rand.NewSource(time.Now().Unix()))
|
||||
f, err := os.CreateTemp(dir, fmt.Sprintf("%s-*", replacer.Replace(tb.Name())))
|
||||
if err != nil {
|
||||
tb.Fatalf("failed to create temp file: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := f.Close(); err != nil {
|
||||
tb.Fatalf("failed to close temp file: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
n, err := random.Read(buf)
|
||||
if err != nil {
|
||||
tb.Fatalf("failed to read random bytes: %v", err)
|
||||
}
|
||||
|
||||
for size > 0 {
|
||||
if size < 1024 {
|
||||
buf = buf[:size]
|
||||
}
|
||||
|
||||
if _, err := f.Write(buf[:n]); err != nil {
|
||||
tb.Fatalf("failed to write random bytes to temp file: %v", err)
|
||||
}
|
||||
|
||||
size -= uint64(len(buf))
|
||||
}
|
||||
|
||||
return f.Name()
|
||||
}
|
||||
|
||||
func createSampleFile(tb testing.TB, size uint64) string {
|
||||
tb.Helper()
|
||||
|
||||
return createSampleFileAt(tb, tb.TempDir(), size)
|
||||
}
|
||||
|
||||
func listFilesInArchive(tb testing.TB, filePath string) (fileList []string) {
|
||||
tb.Helper()
|
||||
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
tb.Fatalf("failed to stat file: %v", err)
|
||||
}
|
||||
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
tb.Fatalf("failed to open tar file: %v", err)
|
||||
}
|
||||
|
||||
peekBuf := make([]byte, 512)
|
||||
read, err := f.ReadAt(peekBuf, 0)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
tb.Fatalf("failed to read tar header: %v", err)
|
||||
}
|
||||
|
||||
switch http.DetectContentType(peekBuf[:read]) {
|
||||
case "application/zip":
|
||||
return listFilesInZip(tb, f, info.Size())
|
||||
case "application/x-gzip":
|
||||
return listFilesInTarGzip(tb, f)
|
||||
case "application/octet-stream":
|
||||
return listFilesInTar(tb, f)
|
||||
default:
|
||||
tb.Fatal("unsupported archive type")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func listFilesInZip(tb testing.TB, reader io.ReaderAt, size int64) (fileList []string) {
|
||||
tb.Helper()
|
||||
|
||||
f, err := zip.NewReader(reader, size)
|
||||
if !assert.NoError(tb, err, "failed to open zip file") {
|
||||
tb.FailNow()
|
||||
}
|
||||
|
||||
for _, file := range f.File {
|
||||
fileList = append(fileList, file.Name)
|
||||
}
|
||||
|
||||
return fileList
|
||||
}
|
||||
|
||||
func listFilesInTarGzip(tb testing.TB, reader io.Reader) (fileList []string) {
|
||||
tb.Helper()
|
||||
|
||||
gzipReader, err := pgzip.NewReader(reader)
|
||||
if !assert.NoError(tb, err, "failed to create gzip reader") {
|
||||
tb.FailNow()
|
||||
}
|
||||
|
||||
return listFilesInTar(tb, gzipReader)
|
||||
}
|
||||
|
||||
func listFilesInTar(tb testing.TB, reader io.Reader) (fileList []string) {
|
||||
tb.Helper()
|
||||
tarReader := tar.NewReader(reader)
|
||||
|
||||
for header, err := tarReader.Next(); err == nil; header, err = tarReader.Next() {
|
||||
fileList = append(fileList, header.Name)
|
||||
}
|
||||
|
||||
return fileList
|
||||
}
|
128
modules/packaging/archive/tar.go
Normal file
128
modules/packaging/archive/tar.go
Normal file
|
@ -0,0 +1,128 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/klauspost/pgzip"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/internal/archive"
|
||||
"code.icb4dc0.de/buildr/buildr/internal/ignore"
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
)
|
||||
|
||||
var (
|
||||
_ modules.Module = (*TarArchive)(nil)
|
||||
_ modules.Helper = (*TarArchive)(nil)
|
||||
)
|
||||
|
||||
type CompressionType string
|
||||
|
||||
func (t CompressionType) Writer(outer io.WriteCloser) (io.WriteCloser, error) {
|
||||
switch t {
|
||||
case CompressionNone:
|
||||
return &noOpWritecloser{Writer: outer}, nil
|
||||
case CompressionGzip:
|
||||
return pgzip.NewWriter(outer), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown compression type: %s", t)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
CompressionNone CompressionType = ""
|
||||
CompressionGzip CompressionType = "gzip"
|
||||
)
|
||||
|
||||
type TarArchive struct {
|
||||
Name string `hcl:"archive_name"`
|
||||
Compression CompressionType `hcl:"compression,optional"`
|
||||
Content []string `hcl:"content,optional"`
|
||||
}
|
||||
|
||||
func (TarArchive) Category() modules.Category {
|
||||
return modules.CategoryPackage
|
||||
}
|
||||
|
||||
func (TarArchive) Type() string {
|
||||
return "tar_archive"
|
||||
}
|
||||
|
||||
func (t TarArchive) Execute(ctx modules.ExecutionContext) (err error) {
|
||||
if t.Name == "" {
|
||||
return ErrEmptyArchiveName
|
||||
}
|
||||
|
||||
logger := ctx.Logger()
|
||||
logger.Info("Packaging archive", slog.String("archive_name", t.Name))
|
||||
|
||||
ignorer, err := ignore.NewIgnorer(ctx.WorkingDir())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ta := archive.Tar{
|
||||
Ignorer: ignorer,
|
||||
}
|
||||
|
||||
if t.Content == nil {
|
||||
if err := t.archiveWorkingDir(&ta, ctx.WorkingDir()); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := t.archiveContent(&ta, ctx.WorkingDir(), logger); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
outFilePath := t.Name
|
||||
if !filepath.IsAbs(t.Name) {
|
||||
outFilePath = filepath.Join(ctx.OutDir(), t.Name)
|
||||
}
|
||||
|
||||
logger.Debug("Creating out file", slog.String("out_file", outFilePath))
|
||||
|
||||
outFile, err := os.Create(outFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create out file for tar archive: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.Join(err, outFile.Close())
|
||||
}()
|
||||
|
||||
compressedWriter, err := t.Compression.Writer(outFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.Join(err, compressedWriter.Close())
|
||||
}()
|
||||
|
||||
return ta.Write(compressedWriter)
|
||||
}
|
||||
|
||||
func (TarArchive) archiveWorkingDir(ta *archive.Tar, workingDir string) error {
|
||||
return ta.Add(workingDir, "")
|
||||
}
|
||||
|
||||
func (t TarArchive) archiveContent(ta *archive.Tar, workingDir string, logger *slog.Logger) error {
|
||||
for _, c := range t.Content {
|
||||
if !filepath.IsAbs(c) {
|
||||
c = filepath.Join(workingDir, c)
|
||||
}
|
||||
|
||||
logger.Debug("Adding file to tar archive", slog.String("file", c))
|
||||
|
||||
if err := ta.Add(c, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
30
modules/packaging/archive/tar_help.go
Normal file
30
modules/packaging/archive/tar_help.go
Normal file
|
@ -0,0 +1,30 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
)
|
||||
|
||||
func (t TarArchive) Help(context.Context) (help modules.Help, err error) {
|
||||
help = modules.Help{
|
||||
Name: t.Type(),
|
||||
Description: `This module helps to create - optionally compressed - tar archives.
|
||||
If there's no explicit ` + "`content`" + ` field, all files in the working directory will be added to the archive.`,
|
||||
Examples: []modules.Example{
|
||||
{
|
||||
Name: "Basic example",
|
||||
Description: `This very basic example explains how to make use of the module.`,
|
||||
Spec: &modules.Metadata[TarArchive]{
|
||||
ModuleName: "buildr_linux_release",
|
||||
Module: TarArchive{
|
||||
Name: "buildr_linux.tar.gz",
|
||||
Compression: CompressionGzip,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return help, nil
|
||||
}
|
156
modules/packaging/archive/tar_test.go
Normal file
156
modules/packaging/archive/tar_test.go
Normal file
|
@ -0,0 +1,156 @@
|
|||
package archive_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
"code.icb4dc0.de/buildr/buildr/modules/packaging/archive"
|
||||
)
|
||||
|
||||
func TestTarArchive_Execute_SingleFileNoContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
archiveName string
|
||||
compression archive.CompressionType
|
||||
}{
|
||||
{
|
||||
name: "Single file - without compression",
|
||||
archiveName: "out.tar",
|
||||
},
|
||||
{
|
||||
name: "Single file - with GZIP compression",
|
||||
archiveName: "out.tar.gz",
|
||||
compression: archive.CompressionGzip,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
workingDir = t.TempDir()
|
||||
outDir = t.TempDir()
|
||||
)
|
||||
|
||||
createSampleFileAt(t, workingDir, 500*1024)
|
||||
|
||||
ta := archive.TarArchive{
|
||||
Name: tt.archiveName,
|
||||
Compression: tt.compression,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
OutputDirectory: outDir,
|
||||
TestWorkingDir: workingDir,
|
||||
}
|
||||
|
||||
if !assert.NoError(t, ta.Execute(execCtx), "failed to execute tar module") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
fileList := listFilesInArchive(t, filepath.Join(outDir, tt.archiveName))
|
||||
assert.Len(t, fileList, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTarArchive_Execute_MultipleFilesWithContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
archiveName string
|
||||
compression archive.CompressionType
|
||||
numberOfFiles int
|
||||
numberOfContentFiles int
|
||||
fileSize uint64
|
||||
}{
|
||||
{
|
||||
name: "Single file and content - plain tar archive",
|
||||
archiveName: "out.tar",
|
||||
numberOfFiles: 1,
|
||||
numberOfContentFiles: 1,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
{
|
||||
name: "Single file and content - tar archive with gzip compression",
|
||||
archiveName: "out.tar.gz",
|
||||
compression: archive.CompressionGzip,
|
||||
numberOfFiles: 1,
|
||||
numberOfContentFiles: 1,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
{
|
||||
name: "Multiple files - sub-selection in content - plain tar",
|
||||
archiveName: "out.tar",
|
||||
numberOfFiles: 5,
|
||||
numberOfContentFiles: 3,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
{
|
||||
name: "Multiple files - sub-selection in content - tar archive with gzip compression",
|
||||
archiveName: "out.tar.gz",
|
||||
compression: archive.CompressionGzip,
|
||||
numberOfFiles: 5,
|
||||
numberOfContentFiles: 3,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
workingDir := t.TempDir()
|
||||
content := make([]string, 0, tt.numberOfContentFiles)
|
||||
|
||||
for i, j := 0, 0; i < 5; i, j = i+1, j+1 {
|
||||
f := createSampleFileAt(t, workingDir, 100*1024)
|
||||
if j < tt.numberOfContentFiles {
|
||||
relFilePath, err := filepath.Rel(workingDir, f)
|
||||
if !assert.NoError(t, err, "failed to get relative path") {
|
||||
t.FailNow()
|
||||
}
|
||||
content = append(content, relFilePath)
|
||||
}
|
||||
}
|
||||
|
||||
ta := archive.TarArchive{
|
||||
Name: tt.archiveName,
|
||||
Compression: tt.compression,
|
||||
Content: content,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
outDir := t.TempDir()
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
OutputDirectory: outDir,
|
||||
TestWorkingDir: workingDir,
|
||||
}
|
||||
|
||||
if !assert.NoError(t, ta.Execute(execCtx), "failed to execute tar module") {
|
||||
t.FailNow()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
|
@ -15,30 +15,38 @@ import (
|
|||
"github.com/klauspost/compress/zip"
|
||||
)
|
||||
|
||||
var _ modules.Module = (*ZipArchive)(nil)
|
||||
var (
|
||||
_ modules.Module = (*ZipArchive)(nil)
|
||||
_ modules.Helper = (*ZipArchive)(nil)
|
||||
)
|
||||
|
||||
type ZipArchive struct {
|
||||
Name string `hcl:"archive_name"`
|
||||
WrapInDir string `hcl:"wrap_in_directory,optional"`
|
||||
Name string `hcl:"archive_name"`
|
||||
WrapInDir string `hcl:"wrap_in_directory,optional"`
|
||||
Content []string `hcl:"content,optional"`
|
||||
}
|
||||
|
||||
func (z ZipArchive) Category() modules.Category {
|
||||
func (ZipArchive) Category() modules.Category {
|
||||
return modules.CategoryPackage
|
||||
}
|
||||
|
||||
func (z ZipArchive) Type() string {
|
||||
func (ZipArchive) Type() string {
|
||||
return "zip_archive"
|
||||
}
|
||||
|
||||
func (z ZipArchive) Execute(ctx modules.ExecutionContext) (err error) {
|
||||
if z.Name == "" {
|
||||
return fmt.Errorf("zip archive name may not be empty")
|
||||
return ErrEmptyArchiveName
|
||||
}
|
||||
|
||||
logger := ctx.Logger()
|
||||
logger.Info("Packaging archive", slog.String("archive_name", z.Name))
|
||||
|
||||
outFilePath := filepath.Join(ctx.OutDir(), z.Name)
|
||||
outFilePath := z.Name
|
||||
if !filepath.IsAbs(z.Name) {
|
||||
outFilePath = filepath.Join(ctx.OutDir(), z.Name)
|
||||
}
|
||||
|
||||
logger.Debug("Creating out file", slog.String("out_file", outFilePath))
|
||||
zipFile, err := os.Create(outFilePath)
|
||||
if err != nil {
|
||||
|
@ -56,13 +64,42 @@ func (z ZipArchive) Execute(ctx modules.ExecutionContext) (err error) {
|
|||
}
|
||||
}()
|
||||
|
||||
if err = filepath.WalkDir(ctx.WorkingDir(), z.walker(logger, zipWriter, ctx.WorkingDir(), z.WrapInDir)); err != nil {
|
||||
if z.Content == nil {
|
||||
return z.compressAllInWorkingDir(logger, zipWriter, ctx.WorkingDir())
|
||||
}
|
||||
|
||||
return z.compressContent(logger, zipWriter, ctx.WorkingDir())
|
||||
}
|
||||
|
||||
func (z ZipArchive) compressAllInWorkingDir(logger *slog.Logger, zipWriter *zip.Writer, workingDir string) error {
|
||||
if err := filepath.WalkDir(workingDir, z.walker(logger, zipWriter, workingDir, z.WrapInDir)); err != nil {
|
||||
return fmt.Errorf("failed to recursively add files to zip archive: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z ZipArchive) compressContent(logger *slog.Logger, zipWriter *zip.Writer, workingDir string) error {
|
||||
for _, srcPath := range z.Content {
|
||||
if !filepath.IsAbs(srcPath) {
|
||||
srcPath = filepath.Join(workingDir, srcPath)
|
||||
}
|
||||
|
||||
info, err := os.Stat(srcPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file info for file %s: %w", srcPath, err)
|
||||
}
|
||||
|
||||
logger.Debug("Adding file to zip archive", slog.String("file", srcPath))
|
||||
|
||||
if err := addFileToArchive(zipWriter, srcPath, workingDir, "", info); err != nil {
|
||||
return fmt.Errorf("failed to add file %s to zip archive: %w", srcPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z ZipArchive) walker(logger *slog.Logger, zipWriter *zip.Writer, basePath, commonPrefix string) fs.WalkDirFunc {
|
||||
return func(currentPath string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
|
@ -74,48 +111,52 @@ func (z ZipArchive) walker(logger *slog.Logger, zipWriter *zip.Writer, basePath,
|
|||
}
|
||||
|
||||
logger.Debug("Adding file to zip archive", slog.String("file", currentPath))
|
||||
archivePath := filepath.ToSlash(currentPath)
|
||||
|
||||
if filepath.IsAbs(archivePath) {
|
||||
if archivePath, err = filepath.Rel(basePath, archivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if commonPrefix != "" {
|
||||
archivePath = path.Join(commonPrefix, archivePath)
|
||||
}
|
||||
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file info for file %s: %w", currentPath, err)
|
||||
}
|
||||
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file header for file %s: %w", currentPath, err)
|
||||
}
|
||||
|
||||
sourceFile, err := os.Open(currentPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file %s: %w", currentPath, err)
|
||||
}
|
||||
|
||||
defer sourceFile.Close()
|
||||
|
||||
header.Method = zip.Deflate
|
||||
header.Name = filepath.ToSlash(archivePath)
|
||||
|
||||
zipFileWriter, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write header for file %s: %w", currentPath, err)
|
||||
}
|
||||
|
||||
_, err = ioutils.CopyWithPooledBuffer(zipFileWriter, sourceFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy file to zip archive %s: %w", currentPath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return addFileToArchive(zipWriter, currentPath, basePath, commonPrefix, info)
|
||||
}
|
||||
}
|
||||
|
||||
func addFileToArchive(zipWriter *zip.Writer, filePath, basePath, commonPrefix string, info fs.FileInfo) (err error) {
|
||||
archivePath := filepath.ToSlash(filePath)
|
||||
if filepath.IsAbs(archivePath) {
|
||||
if archivePath, err = filepath.Rel(basePath, archivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if commonPrefix != "" {
|
||||
archivePath = path.Join(commonPrefix, archivePath)
|
||||
}
|
||||
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file header for file %s: %w", filePath, err)
|
||||
}
|
||||
|
||||
sourceFile, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file %s: %w", filePath, err)
|
||||
}
|
||||
|
||||
defer sourceFile.Close()
|
||||
|
||||
header.Method = zip.Deflate
|
||||
header.Name = filepath.ToSlash(archivePath)
|
||||
|
||||
zipFileWriter, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write header for file %s: %w", filePath, err)
|
||||
}
|
||||
|
||||
_, err = ioutils.CopyWithPooledBuffer(zipFileWriter, sourceFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy file to zip archive %s: %w", filePath, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
30
modules/packaging/archive/zip_help.go
Normal file
30
modules/packaging/archive/zip_help.go
Normal file
|
@ -0,0 +1,30 @@
|
|||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
)
|
||||
|
||||
func (z ZipArchive) Help(context.Context) (help modules.Help, err error) {
|
||||
help = modules.Help{
|
||||
Name: z.Type(),
|
||||
Description: `This module helps to create zip archives.
|
||||
If there's no explicit ` + "`content`" + ` field, all files in the working directory will be added to the archive.
|
||||
Optionally you can wrap the archive in a directory with the ` + "`wrap_in_directory`" + `setting introducing a common prefix.`,
|
||||
Examples: []modules.Example{
|
||||
{
|
||||
Name: "Basic example",
|
||||
Description: `This very basic example explains how to make use of the module.`,
|
||||
Spec: &modules.Metadata[ZipArchive]{
|
||||
ModuleName: "buildr_windows_release",
|
||||
Module: ZipArchive{
|
||||
Name: "buildr_win.zip",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return help, nil
|
||||
}
|
197
modules/packaging/archive/zip_test.go
Normal file
197
modules/packaging/archive/zip_test.go
Normal file
|
@ -0,0 +1,197 @@
|
|||
package archive_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"code.icb4dc0.de/buildr/buildr/modules"
|
||||
"code.icb4dc0.de/buildr/buildr/modules/packaging/archive"
|
||||
)
|
||||
|
||||
func TestZipArchive_Execute_SingleFileNoContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
wrapInDir string
|
||||
}{
|
||||
{
|
||||
name: "Single file - without wrapping in directory",
|
||||
},
|
||||
{
|
||||
name: "Single file - with wrapping in directory",
|
||||
wrapInDir: "out",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
workingDir = t.TempDir()
|
||||
outDir = t.TempDir()
|
||||
)
|
||||
|
||||
createSampleFileAt(t, workingDir, 500*1024)
|
||||
|
||||
zip := archive.ZipArchive{
|
||||
Name: "out.zip",
|
||||
WrapInDir: tt.wrapInDir,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
OutputDirectory: outDir,
|
||||
TestWorkingDir: workingDir,
|
||||
}
|
||||
|
||||
if !assert.NoError(t, zip.Execute(execCtx), "failed to execute zip module") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
outfilePath := filepath.Join(outDir, "out.zip")
|
||||
assert.FileExists(t, outfilePath)
|
||||
assert.Len(t, listFilesInArchive(t, outfilePath), 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestZipArchive_Execute_MultipleFilesNoContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
wrapInDir string
|
||||
numberOfFiles int
|
||||
fileSize uint64
|
||||
}{
|
||||
{
|
||||
name: "Multiple files - without wrapping in directory",
|
||||
numberOfFiles: 5,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
{
|
||||
name: "Multiple files - with wrapping in directory",
|
||||
wrapInDir: "out",
|
||||
numberOfFiles: 5,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
workingDir := t.TempDir()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
createSampleFileAt(t, workingDir, 100*1024)
|
||||
}
|
||||
|
||||
zip := archive.ZipArchive{Name: "out.zip"}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
outDir := t.TempDir()
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
OutputDirectory: outDir,
|
||||
TestWorkingDir: workingDir,
|
||||
}
|
||||
|
||||
if !assert.NoError(t, zip.Execute(execCtx), "failed to execute zip module") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
outFilePath := filepath.Join(outDir, "out.zip")
|
||||
assert.FileExists(t, outFilePath)
|
||||
assert.Len(t, listFilesInArchive(t, outFilePath), tt.numberOfFiles)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestZipArchive_Execute_MultipleFilesWithContent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
wrapInDir string
|
||||
numberOfFiles int
|
||||
numberOfContentFiles int
|
||||
fileSize uint64
|
||||
}{
|
||||
{
|
||||
name: "single file - single content",
|
||||
numberOfFiles: 1,
|
||||
numberOfContentFiles: 1,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
{
|
||||
name: "Multiple files - sub-selection in content",
|
||||
wrapInDir: "out",
|
||||
numberOfFiles: 5,
|
||||
numberOfContentFiles: 3,
|
||||
fileSize: 100 * 1024,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
workingDir := t.TempDir()
|
||||
content := make([]string, 0, tt.numberOfContentFiles)
|
||||
|
||||
for i, j := 0, 0; i < 5; i, j = i+1, j+1 {
|
||||
f := createSampleFileAt(t, workingDir, 100*1024)
|
||||
if j < tt.numberOfContentFiles {
|
||||
relFilePath, err := filepath.Rel(workingDir, f)
|
||||
if !assert.NoError(t, err, "failed to get relative path") {
|
||||
t.FailNow()
|
||||
}
|
||||
content = append(content, relFilePath)
|
||||
}
|
||||
}
|
||||
|
||||
zip := archive.ZipArchive{
|
||||
Name: "out.zip",
|
||||
Content: content,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
outDir := t.TempDir()
|
||||
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
OutputDirectory: outDir,
|
||||
TestWorkingDir: workingDir,
|
||||
}
|
||||
|
||||
if !assert.NoError(t, zip.Execute(execCtx), "failed to execute zip module") {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
outFilePath := filepath.Join(outDir, "out.zip")
|
||||
assert.FileExists(t, outFilePath)
|
||||
assert.Len(t, listFilesInArchive(t, outFilePath), tt.numberOfContentFiles)
|
||||
})
|
||||
}
|
||||
}
|
|
@ -14,4 +14,12 @@ var Registration = modules.RegistrationFunc(func(registry *modules.TypeRegistry)
|
|||
registry.RegisterModule(modules.ModuleFactoryFunc(func() modules.ModuleWithMeta {
|
||||
return new(modules.Metadata[archive.ZipArchive])
|
||||
}))
|
||||
|
||||
registry.RegisterModule(modules.ModuleFactoryFunc(func() modules.ModuleWithMeta {
|
||||
return new(modules.Metadata[archive.TarArchive])
|
||||
}))
|
||||
|
||||
registry.RegisterModule(modules.ModuleFactoryFunc(func() modules.ModuleWithMeta {
|
||||
return new(modules.Metadata[archive.GzipFileCompression])
|
||||
}))
|
||||
})
|
||||
|
|
|
@ -29,11 +29,9 @@ func TestExamples(t *testing.T) {
|
|||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
execCtx := modules.TestExecutionContext{
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
TestWorkingDir: t.TempDir(),
|
||||
OutputDirectory: t.TempDir(),
|
||||
StateStore: &memStateStore,
|
||||
Context: ctx,
|
||||
TB: t,
|
||||
StateStore: &memStateStore,
|
||||
}
|
||||
|
||||
if e.Spec.Unwrap().(task.ScriptTask).WorkingDir != "" {
|
||||
|
|
|
@ -24,11 +24,17 @@ func (t TestExecutionContext) Name() string {
|
|||
}
|
||||
|
||||
func (t TestExecutionContext) WorkingDir() string {
|
||||
return t.TestWorkingDir
|
||||
if t.TestWorkingDir != "" {
|
||||
return t.TestWorkingDir
|
||||
}
|
||||
return t.TB.TempDir()
|
||||
}
|
||||
|
||||
func (t TestExecutionContext) OutDir() string {
|
||||
return t.OutputDirectory
|
||||
if t.OutputDirectory != "" {
|
||||
return t.OutputDirectory
|
||||
}
|
||||
return t.TB.TempDir()
|
||||
}
|
||||
|
||||
func (t TestExecutionContext) BinariesDir() string {
|
||||
|
@ -44,7 +50,11 @@ func (t TestExecutionContext) StdErr() io.Writer {
|
|||
}
|
||||
|
||||
func (t TestExecutionContext) Logger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(testWriter{TB: t.TB}, nil))
|
||||
opts := slog.HandlerOptions{
|
||||
AddSource: true,
|
||||
Level: slog.LevelDebug,
|
||||
}
|
||||
return slog.New(slog.NewTextHandler(testWriter{TB: t.TB}, &opts))
|
||||
}
|
||||
|
||||
func (t TestExecutionContext) GetState(ctx context.Context, key string) ([]byte, state.Metadata, error) {
|
||||
|
|
Loading…
Reference in a new issue