From 25a5d7a0f73570c06aab258b175d14b59be72946 Mon Sep 17 00:00:00 2001 From: Kindritskiy Maksym Date: Mon, 7 Aug 2023 18:12:25 +0300 Subject: [PATCH] add checksum_cmd directive to command --- docs/docs/changelog.md | 67 -------- docs/docs/config.md | 149 +++++++----------- internal/checksum/checksum.go | 15 ++ internal/checksum/checksum_test.go | 19 +++ internal/config/config/command.go | 25 ++- internal/config/config/config_test.go | 22 +++ internal/executor/executor.go | 29 ++-- internal/executor/executor_test.go | 47 ++++++ lets.yaml | 4 + tests/command_checksum_cmd.bats | 40 +++++ tests/command_checksum_cmd/lets.yaml | 12 ++ .../command_checksum_cmd/project/checksum.txt | 1 + tests/command_persist_checksum.bats | 2 +- 13 files changed, 259 insertions(+), 173 deletions(-) create mode 100644 internal/executor/executor_test.go create mode 100644 tests/command_checksum_cmd.bats create mode 100644 tests/command_checksum_cmd/lets.yaml create mode 100644 tests/command_checksum_cmd/project/checksum.txt diff --git a/docs/docs/changelog.md b/docs/docs/changelog.md index 41f61b2f..2a960174 100644 --- a/docs/docs/changelog.md +++ b/docs/docs/changelog.md @@ -5,73 +5,6 @@ title: Changelog ## [Unreleased](https://github.com/lets-cli/lets/releases/tag/v0.0.X) -* `[Dependency]` update go to `1.26` -* `[Added]` Show similar command suggestions on typos. -* `[Changed]` Exit code 2 on unknown command. -* `[Added]` Expose `LETS_OS` and `LETS_ARCH` environment variables at command runtime. -* `[Removed]` Drop deprecated `eval_env` directive. Use `env` with `sh` execution mode instead. -* `[Added]` When a command or its `depends` chain fails, print an indented tree to stderr showing the full chain with the failing command highlighted -* `[Added]` Support `env_file` in global config and commands. File names are expanded after `env` is resolved, and values loaded from env files override values from `env`. -* `[Changed]` Migrate the LSP YAML parser from the CGO-based tree-sitter bindings to pure-Go [`gotreesitter`](https://github.com/odvcencio/gotreesitter), removing the C toolchain requirement from normal builds and release packaging. - -## [0.0.59](https://github.com/lets-cli/lets/releases/tag/v0.0.59) - -* `[Fixed]` Fixed indentation issues for long commands in help output. Command names are now properly padded for consistent alignment. -* `[Refactoring]` Refactored `maxCommandNameLen` to use `slices.MaxFunc` with proper handling for empty command lists. - -## [0.0.58](https://github.com/lets-cli/lets/releases/tag/v0.0.58) - -* `[Added]` `group` directive for commands. Organize commands into groups for better readability in help output. See [config reference for group](/docs/config#group). - - ```yaml - commands: - build: - group: Development - cmd: npm run build - - test: - group: Development - cmd: npm test - - deploy: - group: Operations - cmd: ./deploy.sh - ``` - -## [0.0.57](https://github.com/lets-cli/lets/releases/tag/v0.0.57) - -* `[Dependency]` update go to `1.24` -* `[Added]` support custom top-level keywords that start with `x-` -* `[Added]` check for invalid top-level keywords during config parsing -* `[Added]` support YAML aliases in `env` - env will be merged aliases mapping - -## [0.0.56](https://github.com/lets-cli/lets/releases/tag/v0.0.56) - -This tag is not released due to build issues - -## [0.0.55](https://github.com/lets-cli/lets/releases/tag/v0.0.55) - -* `[Added]` `lets self` command that is ment to be a new home for all lets own commands such as `completions` (soon) or `lsp` -* `[Added]` `lets self lsp` command that starts built-in `lsp` server with go to definition support and completions -* [`Development`] Since `lsp` implementation uses `https://tree-sitter.github.io` (C library with go bindings) as a internal parser `lets` now build with `CGO_ENABLED=1`. If you are building on local machine, you do not have to specify `CGO_ENABLED` env variable. But you may have to install some build system dependencies in case compilatino fails. -* `[CI]` reworked release pipeline now supports go cross compilation -* `[Improvment]` split commands in help message into `Commands` and `Internal commands` -* `[Dependency]` update go to `1.23` -* `[Dependency]` update goreleaser to `1.63.x` -* `[Dependency]` update golangci-lint to `2.x` (also applied some lint fixes across codebase) - -## [0.0.54](https://github.com/lets-cli/lets/releases/tag/v0.0.54) - -* `[Fixed]` `lets --init` now properly creates `lets.yaml`. Issue [#263](https://github.com/lets-cli/lets/issues/263) -* `[Dependency]` update go to `1.22` -* `[Fixed]` ensure `init` script does not get called twice. Issue [#256](https://github.com/lets-cli/lets/issues/256) -* `[Fixed]` do not fail if `sh` in env is empty. Issue [#235](https://github.com/lets-cli/lets/issues/235) -* `[Fixed]` support `arm64` in `lets --upgrade`. Issue [#254](https://github.com/lets-cli/lets/issues/254) - -## [0.0.53](https://github.com/lets-cli/lets/releases/tag/v0.0.53) - -* `[Fixed]` change `SHELL` env to `LETS_SHELL` because setting system variable `SHELL` to just `bash` without full path to binary cases and error in some cases. - ## [0.0.52](https://github.com/lets-cli/lets/releases/tag/v0.0.52) * `[Dependency]` update and pin goreleaser diff --git a/docs/docs/config.md b/docs/docs/config.md index 85387e6a..2289b878 100644 --- a/docs/docs/config.md +++ b/docs/docs/config.md @@ -3,35 +3,25 @@ id: config title: Config reference --- -- [Top-level directives:](#top-level-directives) - - [Version](#version) - - [Shell](#shell) - - [Global env](#global-env) - - [Global before](#global-before) - - [Global init](#global-init) - - [Conditional init](#conditional-init) - - [Mixins](#mixins) - - [Ignored mixins](#ignored-mixins) - - [Remote mixins `(experimental)`](#remote-mixins-experimental) - - [Commands](#commands) -- [Command directives:](#command-directives) - - [Short syntax](#short-syntax) - - [`cmd`](#cmd) - - [`description`](#description) - - [`work_dir`](#work_dir) - - [`shell`](#shell-1) - - [`after`](#after) - - [`depends`](#depends) - - [Override arguments in depends command](#override-arguments-in-depends-command) - - [`options`](#options) - - [`env`](#env) - - [`checksum`](#checksum) - - [`persist_checksum`](#persist_checksum) - - [`ref`](#ref) - - [`args`](#args) - - [`group`](#group) -- [Aliasing:](#aliasing) - - [Env aliasing](#env-aliasing) +* [shell](#shell) +* [mixins](#mixins) +* [env](#global-env) +* [eval_env](#global-eval_env) +* [init](#global-init) +* [before](#global-before) +* [commands](#commands) + * [description](#description) + * [cmd](#cmd) + * [work_dir](#work_dir) + * [after](#after) + * [depends](#depends) + * [options](#options) + * [env](#env) + * [eval_env](#eval_env) + * [checksum](#checksum) + * [persist_checksum](#persist_checksum) + * [ref](#ref) + * [args](#args) ## Top-level directives: @@ -843,6 +833,31 @@ commands: docker run --rm myrepo/app${LETS_CHECKSUM} python -m app ``` +### `checksum_cmd` + +`key: checksum_cmd` + +`type: string` + +Use `checksum_cmd` when checksum should come from a shell command instead of a list of files. + +The command runs with the same effective `shell` and `work_dir` as the command itself, so command-level overrides apply here too. + +Result then can be accessed via `LETS_CHECKSUM` env variable. + +Example: + +```yaml +shell: sh + +commands: + build-image: + shell: bash + work_dir: backend + checksum_cmd: | + [[ -f package-lock.json ]] && sha1sum package-lock.json | cut -d' ' -f1 + cmd: docker build -t myrepo/app:${LETS_CHECKSUM} . +``` ### `persist_checksum` @@ -852,7 +867,7 @@ commands: This feature is useful when you want to know that something has changed between two executions of a command. -`persist_checksum` can be used only if `checksum` declared for command. +`persist_checksum` can be used only if `checksum` or `checksum_cmd` declared for command. If set to `true`, each run all calculated checksums will be stored to disk. @@ -877,6 +892,19 @@ commands: - Readme.md ``` +`checksum_cmd` can be persisted too: + +```yaml +commands: + build-image: + persist_checksum: true + checksum_cmd: git rev-parse HEAD + cmd: | + if [[ ${LETS_CHECKSUM_CHANGED} == true ]]; then + docker build -t myrepo/app:${LETS_CHECKSUM} . + fi +``` + Resulting env will be: * `LETS_CHECKSUM_DEPS` - checksum of deps files @@ -937,64 +965,3 @@ commands: `args` is used only with [ref](#ref) and allows to set additional positional args to referenced command. See [ref](#ref) example. - -### `group` - -`key: group` - -`type: string` - -Commands can be organized into groups for better readability in the help output. To assign a command to a group, use the `group` key: - -```yaml -commands: - build: - group: Build & Deploy - description: Build the project - cmd: npm run build - - deploy: - group: Build & Deploy - description: Deploy the project - cmd: npm run deploy - - test: - group: Testing - description: Run tests - cmd: npm test -``` - -When you run `lets help`, commands will be listed under their respective groups, making it easier to find related commands. - -``` -Commands: - - Build & Deploy - build Build the project - deploy Deploy the project - - Testing - test Run tests -``` - - -## Aliasing: - -Lets supports YAML aliasing in various places in the config - -### Env aliasing - -You can define any mapping and alias it in `env` configuration: - -```yaml -shell: bash - -.default-env: &default-env - FOO: BAR - -env: - <<: *default-env - HELLO: WORLD -``` - -This will merge `env` and `.default-env`. Any environment variables declarations after `<<: ` will override variables defined in aliased map. diff --git a/internal/checksum/checksum.go b/internal/checksum/checksum.go index 851b2ccb..15ea1333 100644 --- a/internal/checksum/checksum.go +++ b/internal/checksum/checksum.go @@ -6,8 +6,10 @@ import ( "encoding/hex" "fmt" "os" + "os/exec" "path/filepath" "sort" + "strings" "github.com/lets-cli/lets/internal/set" "github.com/lets-cli/lets/internal/util" @@ -103,6 +105,19 @@ func getChecksumsKeys(mapping map[string][]string) []string { return keys } +func CalculateChecksumFromCmd(shell string, workDir string, script string) (string, error) { + cmd := exec.Command(shell, "-c", script) + cmd.Dir = workDir + + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("can not calculate checksum from cmd: %s: %w", script, err) + } + + res := string(out) + return strings.TrimSpace(res), nil +} + // CalculateChecksumFromSources calculates checksum from checksumSources. func CalculateChecksumFromSources(workDir string, checksumSources map[string][]string) (map[string]string, error) { checksumMap := make(map[string]string) diff --git a/internal/checksum/checksum_test.go b/internal/checksum/checksum_test.go index d7a221ce..53bf2df4 100644 --- a/internal/checksum/checksum_test.go +++ b/internal/checksum/checksum_test.go @@ -3,6 +3,7 @@ package checksum import ( "fmt" "os" + "path/filepath" "testing" "github.com/lets-cli/lets/internal/test" @@ -136,3 +137,21 @@ func TestCalculateChecksumFromListOrMap(t *testing.T) { ) } } + +func TestCalculateChecksumFromCmdUsesWorkDir(t *testing.T) { + tempDir := t.TempDir() + checksumFilePath := filepath.Join(tempDir, "checksum.txt") + + if err := os.WriteFile(checksumFilePath, []byte("checksum-from-workdir"), 0o600); err != nil { + t.Fatalf("can not write checksum file. Error: %s", err) + } + + checksumResult, err := CalculateChecksumFromCmd("sh", tempDir, "cat checksum.txt") + if err != nil { + t.Fatalf("checksum command failed. Error: %s", err) + } + + if checksumResult != "checksum-from-workdir" { + t.Fatalf("wrong checksum output. Expect: %s, got: %s", "checksum-from-workdir", checksumResult) + } +} diff --git a/internal/config/config/command.go b/internal/config/config/command.go index da15fb59..79afa989 100644 --- a/internal/config/config/command.go +++ b/internal/config/config/command.go @@ -36,6 +36,7 @@ type Command struct { Depends *Deps ChecksumMap map[string]string PersistChecksum bool + ChecksumCmd string // args from 'lets run --debug' will become [--debug] Args []string @@ -73,7 +74,8 @@ func (c *Command) UnmarshalYAML(unmarshal func(any) error) error { After string Ref string Checksum *Checksum - PersistChecksum bool `yaml:"persist_checksum"` + ChecksumCmd string `yaml:"checksum_cmd"` + PersistChecksum bool `yaml:"persist_checksum"` } if err := unmarshal(&cmd); err != nil { @@ -120,9 +122,13 @@ func (c *Command) UnmarshalYAML(unmarshal func(any) error) error { c.ChecksumSources = *cmd.Checksum } + if cmd.ChecksumCmd != "" { + c.ChecksumCmd = cmd.ChecksumCmd + } + c.PersistChecksum = cmd.PersistChecksum - if len(c.ChecksumSources) == 0 && c.PersistChecksum { - return errors.New("'persist_checksum' must be used with 'checksum'") + if len(c.ChecksumSources) == 0 && c.ChecksumCmd == "" && c.PersistChecksum { + return errors.New("'persist_checksum' must be used with 'checksum' or 'checksum_cmd'") } if cmd.Ref != "" { @@ -189,6 +195,7 @@ func (c *Command) Clone() *Command { Depends: c.Depends.Clone(), ChecksumMap: cloneMap(c.ChecksumMap), PersistChecksum: c.PersistChecksum, + ChecksumCmd: c.ChecksumCmd, ChecksumSources: cloneMapSlice(c.ChecksumSources), persistedChecksums: cloneMap(c.persistedChecksums), Args: cloneSlice(c.Args), @@ -227,7 +234,17 @@ func (c *Command) Help() string { return strings.TrimSuffix(buf.String(), "\n") } -func (c *Command) ChecksumCalculator(workDir string) error { +func (c *Command) ChecksumCalculator(shell, workDir string) error { + if c.ChecksumCmd != "" { + checksumResult, err := checksum.CalculateChecksumFromCmd(shell, workDir, c.ChecksumCmd) + if err != nil { + return err + } + c.ChecksumMap = make(map[string]string, 1) + c.ChecksumMap[checksum.DefaultChecksumKey] = checksumResult + return nil + } + if len(c.ChecksumSources) == 0 { return nil } diff --git a/internal/config/config/config_test.go b/internal/config/config/config_test.go index aec94442..c3197cbf 100644 --- a/internal/config/config/config_test.go +++ b/internal/config/config/config_test.go @@ -88,4 +88,26 @@ func TestParseConfig(t *testing.T) { t.Errorf("config must not allow custom keywords") } }) + + t.Run("allow persist checksum with checksum cmd", func(t *testing.T) { + text := dedent.Dedent(` + shell: bash + commands: + checksum-cmd: + persist_checksum: true + checksum_cmd: echo checksum + cmd: echo ok + `) + + cfg := ConfigFixture(t, text) + cmd := cfg.Commands["checksum-cmd"] + + if !cmd.PersistChecksum { + t.Fatalf("expected persist_checksum to be enabled") + } + + if cmd.ChecksumCmd != "echo checksum" { + t.Fatalf("wrong checksum_cmd. Expect: %s, got: %s", "echo checksum", cmd.ChecksumCmd) + } + }) } diff --git a/internal/executor/executor.go b/internal/executor/executor.go index 9c5cc08b..be41ac66 100644 --- a/internal/executor/executor.go +++ b/internal/executor/executor.go @@ -156,6 +156,22 @@ func formatOptsUsageError(err error, opts docopt.Opts, cmdName string, rawOption return fmt.Errorf("%s\n\n%s", errTpl, rawOptions) } +func (e *Executor) shellForCommand(command *config.Command) string { + if command.Shell != "" { + return command.Shell + } + + return e.cfg.Shell +} + +func (e *Executor) workDirForCommand(command *config.Command) string { + if command.WorkDir != "" { + return command.WorkDir + } + + return e.cfg.WorkDir +} + // Init Command before execution: // - parse docopt // - calculate checksum. @@ -179,7 +195,7 @@ func (e *Executor) initCmd(ctx *Context) error { } // calculate checksum if needed - if err := cmd.ChecksumCalculator(e.cfg.WorkDir); err != nil { + if err := cmd.ChecksumCalculator(e.shellForCommand(cmd), e.workDirForCommand(cmd)); err != nil { return fmt.Errorf("failed to calculate checksum for command '%s': %w", cmd.Name, err) } @@ -254,11 +270,7 @@ func (e *Executor) setupEnv(osCmd *exec.Cmd, command *config.Command, shell stri // Passing ctx will change behavior of program drastically - it will kill process if context will be canceled. func (e *Executor) newOsCommand(command *config.Command, cmdScript string) (*exec.Cmd, error) { script := joinBeforeAndScript(e.cfg.Before, cmdScript) - - shell := e.cfg.Shell - if command.Shell != "" { - shell = command.Shell - } + shell := e.shellForCommand(command) args := []string{"-c", script} if len(command.Args) > 0 { @@ -277,10 +289,7 @@ func (e *Executor) newOsCommand(command *config.Command, cmdScript string) (*exe osCmd.Stdin = os.Stdin // set working directory for command - osCmd.Dir = e.cfg.WorkDir - if command.WorkDir != "" { - osCmd.Dir = command.WorkDir - } + osCmd.Dir = e.workDirForCommand(command) if err := e.setupEnv(osCmd, command, shell); err != nil { return nil, err diff --git a/internal/executor/executor_test.go b/internal/executor/executor_test.go new file mode 100644 index 00000000..70a12705 --- /dev/null +++ b/internal/executor/executor_test.go @@ -0,0 +1,47 @@ +package executor + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/lets-cli/lets/internal/checksum" + "github.com/lets-cli/lets/internal/config/config" +) + +func TestInitCmdUsesCommandShellAndWorkDirForChecksum(t *testing.T) { + tempDir := t.TempDir() + projectDir := filepath.Join(tempDir, "project") + + if err := os.Mkdir(projectDir, 0o755); err != nil { + t.Fatalf("can not create project dir: %s", err) + } + + if err := os.WriteFile(filepath.Join(projectDir, "checksum.txt"), []byte("command-checksum"), 0o600); err != nil { + t.Fatalf("can not write checksum file: %s", err) + } + + cfg := &config.Config{ + Shell: "sh", + WorkDir: tempDir, + } + cmd := &config.Command{ + Name: "checksum-cmd", + Shell: "bash", + WorkDir: projectDir, + SkipDocopts: true, + ChecksumCmd: "[[ -f checksum.txt ]] && cat checksum.txt", + } + + executor := NewExecutor(cfg, nil) + ctx := NewExecutorCtx(context.Background(), cmd) + + if err := executor.initCmd(ctx); err != nil { + t.Fatalf("initCmd failed: %s", err) + } + + if got := cmd.ChecksumMap[checksum.DefaultChecksumKey]; got != "command-checksum" { + t.Fatalf("wrong checksum output. Expect: %s, got: %s", "command-checksum", got) + } +} diff --git a/lets.yaml b/lets.yaml index cefd782d..d26b10d6 100644 --- a/lets.yaml +++ b/lets.yaml @@ -140,3 +140,7 @@ commands: run-docs: work_dir: docs cmd: npm start + + x: + checksum_cmd: echo xxx_checksum + cmd: echo checksum for x is ${LETS_CHECKSUM} \ No newline at end of file diff --git a/tests/command_checksum_cmd.bats b/tests/command_checksum_cmd.bats new file mode 100644 index 00000000..01d6a1a0 --- /dev/null +++ b/tests/command_checksum_cmd.bats @@ -0,0 +1,40 @@ +load test_helpers + +reset_test_files() { + cleanup + printf "first-checksum" > project/checksum.txt +} + +setup() { + load "${BATS_UTILS_PATH}/bats-support/load.bash" + load "${BATS_UTILS_PATH}/bats-assert/load.bash" + cd ./tests/command_checksum_cmd + reset_test_files +} + +teardown() { + reset_test_files +} + +@test "command_checksum_cmd: should use command shell and work_dir and persist checksum" { + run lets checksum-cmd + + assert_success + assert_line --index 0 "LETS_CHECKSUM=first-checksum" + assert_line --index 1 "LETS_CHECKSUM_CHANGED=true" + [[ -f .lets/checksums/checksum-cmd/lets_default_checksum ]] + + run lets checksum-cmd + + assert_success + assert_line --index 0 "LETS_CHECKSUM=first-checksum" + assert_line --index 1 "LETS_CHECKSUM_CHANGED=false" + + printf "second-checksum" > project/checksum.txt + + run lets checksum-cmd + + assert_success + assert_line --index 0 "LETS_CHECKSUM=second-checksum" + assert_line --index 1 "LETS_CHECKSUM_CHANGED=true" +} diff --git a/tests/command_checksum_cmd/lets.yaml b/tests/command_checksum_cmd/lets.yaml new file mode 100644 index 00000000..f5061928 --- /dev/null +++ b/tests/command_checksum_cmd/lets.yaml @@ -0,0 +1,12 @@ +shell: sh + +commands: + checksum-cmd: + shell: bash + work_dir: project + persist_checksum: true + checksum_cmd: | + [[ -f checksum.txt ]] && cat checksum.txt + cmd: | + echo LETS_CHECKSUM=${LETS_CHECKSUM} + echo LETS_CHECKSUM_CHANGED=${LETS_CHECKSUM_CHANGED} diff --git a/tests/command_checksum_cmd/project/checksum.txt b/tests/command_checksum_cmd/project/checksum.txt new file mode 100644 index 00000000..00fffd6b --- /dev/null +++ b/tests/command_checksum_cmd/project/checksum.txt @@ -0,0 +1 @@ +first-checksum diff --git a/tests/command_persist_checksum.bats b/tests/command_persist_checksum.bats index fc0074e2..bc8b15be 100644 --- a/tests/command_persist_checksum.bats +++ b/tests/command_persist_checksum.bats @@ -143,5 +143,5 @@ TEMP_FILE=foo_test.txt [[ $status = 1 ]] - assert_line --index 0 "lets: config error: failed to parse lets.yaml: 'persist_checksum' must be used with 'checksum'" + assert_line --index 0 "lets: config error: failed to parse lets.yaml: 'persist_checksum' must be used with 'checksum' or 'checksum_cmd'" }