#17089 python_source/python_sources, restartable=True Issue created by thelittlebug with the new run_goal_use_sandbox is there a way to make restartable usable too? atm it only works on pex_binary but thats slowing down the dev iterations massively. ref: https://app.slack.com/client/T046T6T8L/C046T6T9U/thread/C046T6T9U-1664737277.432239 pantsbuild/pants

#17418 Option to generate lockfiles lazily New issue created by da-tubi Is your feature request related to a problem? Please describe. https://github.com/da-tubi/jupyterlab-best-practice I tried to make a jupyterlab-best-practice, the key is

bin/lab xyz

to launch the jupyterlab. But when I'm changing the Python dependencies, I have to re-generate the lockfiles. As we all know,

generate-lockfiles

is a slow operation. It is not a good idea for me to put

./pants generate-lockfiles --resolve=xyz

into

bin/lab

. Describe the solution you'd like I need a

--lazy

option for

generate-lockfiles

, like

./pants generate-lockfiles --lazy --resolve=xyz

If the dependencies didn't change, it should be very fast. Describe alternatives you've considered None Additional context None pantsbuild/pants

#16949 Pants 2.14.x stopped inferring dependency between `shunit2_test` and `shell_sources` Issue created by danxmoran Describe the bug I upgraded from Pants 2.13.0 to 2.14.0rc0. When I did, a

shunit2_test

started failing because the

shell_source

it uses was no longer present in the sandbox - I had to add an explicit dependency to get the test passing again. The directory layout is:

build/pants/ci/
  BUILD.pants
  common.sh
  test_common.sh

With `BUILD.pants`:

shell_sources()

shunit2_tests(name="tests", shell="bash")

And `common.sh`:

# shellcheck shell=bash
set -euo pipefail

# Run a pants goal, selecting targets using environment variables.
# Expects TARGETS to be set to 'all' or 'changed'. If ALLOW_EXPLICIT_TARGETS=true you
# can also put a space delimited list of targets in TARGETS.
#
# When calling this function, any additional args you want to set on your goal
# must be set via env vars.
#
# Usage example:
#   source build/pants/ci/common.sh
#   PANTS_FILTER_TARGET_TYPE=python_test,shunit2_test run_pants test
run_pants() {
    if [[ $# -ne 1 ]]; then
        echo "::error::Usage: run_pants <goal>" >&2
        return 1
    fi
    local -r goal="$1"

    if [[ -z "${PANTS_LOG:-}" ]]; then
        PANTS_LOG=/dev/null
    fi

    local -a pants_args=("$goal")

    if [[ "${TARGETS:-}" == "changed" ]]; then
        if [[ -z ${PANTS_CHANGED_SINCE:-} ]]; then
            echo "::error::Must set PANTS_CHANGED_SINCE if TARGETS=changed" >&2
            return 1
        fi
        if [[ -z ${PANTS_CHANGED_DEPENDEES:-} ]]; then
            echo "::error::Must set PANTS_CHANGED_DEPENDEES if TARGETS=changed" >&2
            return 1
        fi
    elif [[ "${TARGETS:-}" == "all" ]]; then
        if [[ -n ${PANTS_CHANGED_SINCE:-} ]]; then
            echo "::warning::TARGETS=all, unsetting PANTS_CHANGED_SINCE" >&2
            unset PANTS_CHANGED_SINCE
        fi
        pants_args=("${pants_args[@]}" ::)
    elif [[ "${ALLOW_EXPLICIT_TARGETS:-false}" == "true" ]]; then
        if [[ -z ${TARGETS:-} ]]; then
            echo "::error::Must set TARGETS={changed,all,space delimited list of targets}" >&2
            return 1
        fi
        # Expect TARGETS to be a space delimited list of pants targets.
        # shellcheck disable=SC2034
        readarray -td '' targets < <(xargs -n1 printf "%s\0" <<< "$TARGETS")
        pants_args=("${pants_args[@]}" "${targets[@]}")
    else
        echo "::error::Must explicitly set TARGETS={changed,all}" >&2
        return 1
    fi

    attempts=1
    while [ "$attempts" -le 3 ]; do
        if ./pants "${pants_args[@]}" |& tee "${PANTS_LOG}"; then
            return 0
        fi

        # We see <https://github.com/pantsbuild/pants/issues/16778> a few times per day, and have
        # hit a wall with debugging. Work around it for now by retrying.
        if grep 'ImportError: No module named __pants_df_parser' "${PANTS_LOG}"; then
            echo "::warning::Hit ImportError on __pants_df_parser, attempt #${attempts}"
            rm -r "${PANTS_NAMED_CACHES_DIR}/pex_root"
            attempts=$((attempts + 1))
            continue
        fi
        if grep -E "${PANTS_NAMED_CACHES_DIR}/pex_root/venvs/[a-f0-9]+/[a-f0-9]+/pex.+No such file or directory" "${PANTS_LOG}"; then
            echo "::warning::Hit 'No such file' error running 'pex', attempt #${attempts}"
            rm -r "${PANTS_NAMED_CACHES_DIR}/pex_root"
            attempts=$((attempts + 1))
            continue
        fi

        # If we reach this point, we failed for some reason other than the ImportError bug.
        break
    done

    return 1
}

And `test_common.sh`:

#!/usr/bin/env bash

oneTimeSetUp() {
    source build/pants/ci/common.sh
}

setUp() {
    cat << 'EOF' > ./pants
#!/usr/bin/env bash
echo PANTS_CHANGED_SINCE=${PANTS_CHANGED_SINCE}
echo PANTS_CHANGED_DEPENDEES=${PANTS_CHANGED_DEPENDEES}
echo Args: ${@}
EOF
    chmod +x pants
}

testRunPantsMultiArg() {
    run_pants foo bar
    assertEquals 1 $?
}

testRunPantsTargetsAll() {
    mapfile -t output < <(TARGETS=all run_pants lint)
    assertEquals 'PANTS_CHANGED_SINCE=' "${output[0]}"
    assertEquals 'PANTS_CHANGED_DEPENDEES=' "${output[1]}"
    assertEquals 'Args: lint ::' "${output[2]}"

    PANTS_CHANGED_SINCE=master TARGETS=all run_pants lint
    # PANTS_CHANGED_SINCE should be ignored
    assertEquals 0 $?
}

testRunPantsTargetsChanged() {
    TARGETS=changed run_pants lint
    assertEquals 1 $?

    PANTS_CHANGED_SINCE=master TARGETS=changed run_pants lint
    assertEquals 1 $?

    mapfile -t output < <(
        PANTS_CHANGED_SINCE=master PANTS_CHANGED_DEPENDEES=direct TARGETS=changed \
            run_pants lint
    )
    assertEquals 'PANTS_CHANGED_SINCE=master' "${output[0]}"
    assertEquals 'PANTS_CHANGED_DEPENDEES=direct' "${output[1]}"
    assertEquals 'Args: lint' "${output[2]}"
}

testRunPantsExplicitTargets() {
    TARGETS="foo bar" run_pants lint
    assertEquals 1 $?

    ALLOW_EXPLICIT_TARGETS=true run_pants lint
    assertEquals 1 $?

    mapfile -t output < <(
        TARGETS="foo bar" ALLOW_EXPLICIT_TARGETS=true run_pants lint
    )
    assertEquals 'PANTS_CHANGED_SINCE=' "${output[0]}"
    assertEquals 'PANTS_CHANGED_DEPENDEES=' "${output[1]}"
    assertEquals 'Args: lint foo bar' "${output[2]}"
}

Pants version 2.14.0rc0 (and 2.14.0rc1, and the current HEAD of the 2.14.x branch). OS Both pantsbuild/pants

#17426 Synthetic targets should not live outside of existing address namespaces Issue created by kaos In order to avoid confusion, synthetic targets must exist within known address namespaces so they are included in

AllTargets

. pantsbuild/pants

#17427 External go `main` packages should be runnable New issue created by tgolsson Is your feature request related to a problem? Please describe. I'm trying to use https://github.com/kubernetes-sigs/controller-tools as a code-generator without building a whole sub-system around it1. I noticed that this isn't currently possible to do, as I can't run a binary in an external repository. Describe the solution you'd like I'd like the external package to be runnable. If

<http://github.com/org/repo|github.com/org/repo>

has a

cmd/hello

which is a main package I would want

pants run //:root#<http://github.com/org/repo/cmd/hello|github.com/org/repo/cmd/hello>

to run it, at a minimum. This is unambiguous because that cannot be imported, so there's no need for it to be a package. This'd entail replacing or supplementing the current

go_third_party_package

with a

go_binary

target. I'd give the binary target priority here since the

go_third_party_package

doesn't provide any useful actions. While a bit out of scope for the immediate feature ask (which I've offered to tackle), I think there's also other concerns here. Making

package main

a valid "package" in the build graph seems like it potentially can lead to unexpected behaviors. Right now you can depend on the external main package in your build files without it being used. I've tried to enumerate a few different combos here: https://github.com/tgolsson/example-pants-first-party/blob/main/pkg/BUILD. The first ones work as expected, and the last bunch work despite being "ill-formed" in various ways. I think from a discoverability perspective it'd make more sense (to me!) to delineate the based on what kind of package it is to go. So runnable packages are disjoint from importable packages, not just wrappers. Go already knows this difference... and tailor does for the local workspace. As mentioned, properly out of scope... but I had 2 cents and I'd hate to waste them. 😂 Describe alternatives you've considered Not really solvable at all currently. The minimal work is just making

go_binary

accept a

go_third_party_package

as well as a

go_package

. Better/more explicit design is a lot more work (see discussion above). Additional context Slack question here: https://pantsbuild.slack.com/archives/C046T6T9U/p1667252451022069 Footnotes 1. I don't know if I can build a subsystem around a go package; but I assume it'd require a lot of plumbing at a minimum. :leftwards_arrow_with_hook: pantsbuild/pants

#17428 `pex_binary` needs an `environment=` field Issue created by stuhood pantsbuild/pants

#16601 We neither merge nor error when PYTEST_ADDOPTS is specified in user config. Issue created by jsirois A user found when they ran certain tests, this was logged:

[WARN] Failed to generate JUnit XML data for test/python/...

Those tests have a BUILD of:

python_tests(
    name="tests",
    dependencies=[
        "3rdparty/python:nautobot-plugin-reqs",
        "3rdparty/python:reqs#pytest-django",
        "src/python/nautobot_animal_sounds",
        "src/python/netops_nautobot",
        "src/python/pytest_nautobot",
    ],
    extra_env_vars=[
        "DJANGO_SETTINGS_MODULE=nautobot.core.tests.nautobot_config",
        "NAUTOBOT_CONFIG=./src/python/netops_nautobot/nautobot_config.py",
        "NAUTOBOT_REDIS_PASSWORD=decinablesprewad",
        "PYTEST_ADDOPTS=-p pytest_nautobot --reuse-db",
    ],
    sources=["**/test_*.py"],
)

And we do not guard against

PYTEST_ADDOPTS

being set by the user; so our communication channel with pytest is stomped: pants/src/python/pants/backend/python/goals/pytest_runner.py Lines 315 to 322 in </pantsbuild/pants/commit/a69b8f0ef4e14451dbf7f39fd23adaf3f2700535|a69b8f0> pantsbuild/pants

#1971 Pex POSIX locking appears to be broken. New issue created by jsirois I added a knob to force bsd style locks using

_PEX_LOCK_FILE_STYLE=bsd

in #1962 and this looks like it solved #1969. The failure mechanism of POSIX locks needs to be understood and either bsd locks switched to wholesale or else POSIX locks fixed. The

_PEX_LOCK_FILE_STYLE

env var should then be deprecated to emit warning messages on use and then removed in 3.0.0. pantsbuild/pex

#13201 go: add coverage analysis to running tests Issue created by tdyas The

go test

command supports coverage analysis via the

-cover

,

-covermode

, and

-coverpkg

options. We should implement similar coverage analysis for

./pants test

. Help text:

-cover
            Enable coverage analysis.
            Note that because coverage works by annotating the source
            code before compilation, compilation and test failures with
            coverage enabled may report line numbers that don't correspond
            to the original sources.

        -covermode set,count,atomic
            Set the mode for coverage analysis for the package[s]
            being tested. The default is "set" unless -race is enabled,
            in which case it is "atomic".
            The values:
                set: bool: does this statement run?
                count: int: how many times does this statement run?
                atomic: int: count, but correct in multithreaded tests;
                        significantly more expensive.
            Sets -cover.

        -coverpkg pattern1,pattern2,pattern3
            Apply coverage analysis in each test to packages matching the patterns.
            The default is for each test to analyze only the package being tested.
            See 'go help packages' for a description of package patterns.
            Sets -cover.

pantsbuild/pants

#17437 go: support enabling data race detector Issue created by tdyas

go

supports a

-race

argument to build binaries and tests with a runtime data race detector. Pants should support it as well via a field on

go_binary

and

go_package

(for tests) to enable/disable the data race detector. There would be option(s) to set the default value of the field if it were not specified. pantsbuild/pants

#17438 go: support enabling additional build tags New issue created by tdyas

go

supports enabling extra build tags while building via the

-tags

argument to

go build

. This behavior should be made available in Pants. Could be modeled as a

build_tags

field on

go_binary

. pantsbuild/pants

#17439 go: support passing arbitrary compiler flags New issue created by tdyas

go build

accepts a

-gcflags

argument to pass flags to the underlying Go compiler invocations (i.e.,

go tool compile

). Pants should support a mechanism to do that as well. Model as fields on

go_binary

and

go_package

(for tests) to add compiler options with defaults provided via option(s). pantsbuild/pants

#17440 go: support passing arbitrary linker flags New issue created by tdyas

go

supports passing arbitrary linker flags to

go tool link

invocations via a

-ldflags

argument. Pants should support this feature. Model as fields on

go_binary

and

go_package

(for tests) that set additional linker flags to use with defaults coming from option(s). pantsbuild/pants

#17441 go: support passing arbitrary assembler flags New issue created by tdyas

go

supports passing arbitrary assembler flags to

go tool asm

invocations via a

-asmflags

argument. Pants should support this feature. Model as fields on

go_binary

and

go_package

(for tests) that set additional assembler flags to use with defaults coming from option(s). pantsbuild/pants

#17443 go: supporting building Go plugins New issue created by tdyas

go

supports building Go plugins which are Go packages that can be loaded dynamically by a Go binary via the

-buildmode=plugin

flag. Model either as a field on

go_binary

or by a new

go_plugin

target type (which would use same build-related fields as

go_binary

). pantsbuild/pants

#17444 go: supporting build C archives from Go binaries New issue created by tdyas

go

supports building C archives and C shared libraries from Go main packages via the

-buildmode=c-archive

and

-buildmode=c-shared

flags. Model as new target types

go_c_library

and

go_c_shared_library

target types (or other bike-shedded names). pantsbuild/pants

#17445 go: support building Go shared libraries New issue created by tdyas go supports building Go shared libraries from Go main packages via the -buildmode=shared flag. Model as new target type

go_shared_library

(or other bike-shedded names). pantsbuild/pants

#17446 go: use target spec in output path for coverage New issue created by tdyas Use target spec as part of output path for coverage results instead of import path with

/

->`_` conversion. Would match better with how other output is generated under

dist/

. pantsbuild/pants

#17447 Stabilization Tracking: Golang New issue created by tdyas This is a stabilization tracking ticket for the Go backend. Items here are intended to be resolved in order to remove the "experimental" label from Go support: For stabilization Build: ☐ #12768 ☑︎ #16827 ☑︎ #12943 ☐ #17478 • Build options: ☑︎ #16833 ☑︎ #17439 ☑︎ #17440 ☐ #17441 ☑︎ #17437 ☑︎ #16838 ☑︎ #16837 ☐ #17442 Test: ☐ #16610 ☐ #16609 ☑︎ #17260 ☐ #13782 ☐ #17446 Not for stabilization • #13411 • #16217 • #13784 • #16842 • #17438 • #17443 • #17444 • #17445 pantsbuild/pants

#17450 Clarify that `__defaults__` apply before target generation in docs Issue created by danxmoran The docs for

__defaults__

don't mention how the feature interacts with target generation (i.e. if defaults for

python_test

will also apply to targets generated from a

python_tests

). According to @kaos, default-application happens before target generation, so you have to specify

__defaults__

for both the generator and generated target types. It would be good to make that explicit in the docs. pantsbuild/pants

#1973 Allow system_site_packages=True in Virtualenv.create() New issue created by PhoenixHe-TSP Currently PEX could create virtualenv to populate bundled packages, and in PEX code it allows to create venv with "--system-site-packages", but there is no way to specify this parameter at runtime. We want to run with venv and system-site-packages enabled, can we have an extra environment variable to specify this behavior? pantsbuild/pex

#17456 re-add lazy upload of input root for remote execution New issue created by tdyas The REAPI client used to support lazily uploading digests by only uploading them after they were identified as missing in an response to an execution request. See pants/src/rust/engine/process_execution/src/remote.rs Lines 560 to 594 in </pantsbuild/pants/commit/4050ad01a5d1624823548fa5ba6cff960415064e|4050ad0> . More recent changes to Pants have made the upload eager. See #14723 and pants/src/rust/engine/process_execution/src/remote.rs Lines 1547 to 1553 in </pantsbuild/pants/commit/7eb418038e08346850a429043bca1d259cb6cbdb|7eb4180> . Pants should switch back to lazy uploads of the input root and relying on the server to notify Pants when uploads are needed. pantsbuild/pants

#17458 New default of `pull=True` on `docker_image` target breaks use of transitive base images Issue created by danxmoran Describe the bug (Sorry for the wordy title, couldn't think of a better way to put it) In our repo, we have a "base"

Dockerfile

containing common build steps. It's a lightweight image, so we don't push it anywhere - instead, all images that use it build off of it using:

ARG BASE=src/infra/docker/color-base-python:color-base-python
FROM $BASE

On Pants 2.14, if I

./pants package some/top-level/Dockerfile

then Pants will see the

FROM

dependency and build

src/infra/docker/color-base-python

first. When trying out Pants 2.15.0a0 this no longer happens, so the top-level

package

fails with:

#3 [internal] load metadata for <http://docker.io/color/color-base-python:latest|docker.io/color/color-base-python:latest>
#3 ERROR: pull access denied, repository does not exist or may require authorization: server message: insufficient_scope: authorization failed
------
 > [internal] load metadata for <http://docker.io/color/color-base-python:latest|docker.io/color/color-base-python:latest>:
------
ERROR: failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to create LLB definition: pull access denied, repository does not exist or may require authorization: server message: insufficient_scope: authorization failed

./pants dependencies

of the top-level image still shows

src/infra/docker/color-base-python:color-base-python

so dependency inference still works, but something in the sequencing of

docker build

commands has broken. Pants version v2.15.0a0 OS Both Additional info https://app.toolchain.com/organizations/color/repos/color/builds/pants_run_2022_11_03_12_17_40_671_7f0821ce4406484b85bbbd6bf52264e0/type/goal/?subTab=package pantsbuild/pants

#1975 Release 2.1.114 New issue created by jsirois On the docket: ☐ Only insert "" to head of sys.path if a venv PEX runs in interpreter mode #1984 ☑︎ venv_dir calculation doesn't correctly handle PEX_PYTHON_PATH with symlinks. #1885 pantsbuild/pex

#1958 Release 2.1.113 New issue created by jsirois On the docket: ☑︎ Restore AtomicDirectory non-locked good behavior. #1974 pantsbuild/pex

#13259 Support for shading deploy jars Issue created by stuhood #12796 added support for building deploy ("fat") jars. Most usecases of deploy jars also need the ability to shade the contents of the jar (in order to deploy it to Hadoop, Spark, or other environments without classloader isolation). Most likely this will involve using 9and likely resurrecting) our fork of `jarjar`... but it's also possible that modern alternatives have emerged, so we should look at what other tools are doing first. pantsbuild/pants

#17470 Weird stdout redirection behavior Issue created by benjyw

$ ./pants version > version.stdout 2> version.stderr
$ wc -l version.stdout 
   1 version.stdout
$ wc -l version.stderr
   0 version.stderr
$ # Makes sense, version emits to stdout
$
$ ./pants version > version.stdout 
$ wc -l version.stdout 
   0 version.stdout
$ wc -l version.stderr
   0 version.stderr
$ # What? Where did the output go?

This is due to #16822, although it is unclear why. Reverting that change fixes this. Note that it doesn't reproduce when running pants in an integration test. pantsbuild/pants

#17477 batching dep inference New discussion created by benjyw I did a little bit of manual benchmarking of the python dependency parser process. I took that script, modified it to accept multiple input files, and ran it on all 1227 .py files under src/python/pants, with various size batches. As expected, almost the entire runtime consists of process overhead: Note that I ran this experiment outside of Pants, using

find

and

xargs

, so this doesn't include sandbox setup time. Also, the batches ran sequentially, so the wall time of the current one-file-per-process strategy in Pants is faster than that 99 seconds, thanks to parallel execution. But it's still much slower and more CPU-expensive than it needs to be. E.g., we know we have users with larger repos for whom full-repo dep inference (e.g., in a call to

./pants peek

) takes several minutes. So it seems reasonably clear that batching the dependency parsing is a big perf win. (I'm referring to Python here, it seems likely that similar benefits would obtain for JVM at least). This discussion is to, er, discuss some options for doing so. pantsbuild/pants

#17478 go: copy header files named for current GOOS/GOARCH to standard names New issue created by tdyas Copy .h files named for goos or goarch or goos_goarch to names using GOOS and GOARCH. For example,

defs_linux_amd64.h

becomes

defs_GOOS_GOARCH.h

. See https://github.com/golang/go/blob/1c05968c9a5d6432fc6f30196528f8f37287dd3d/src/cmd/go/internal/work/exec.go#L867-L892. pantsbuild/pants

#17482 subsystem flag leaking process environment Issue created by davidreuss Describe the bug I have a project where i use

--env

as a flag name for my subsystem. That resulted in the current calculated value (even when a string default is set) is set to the full dump of my local process environment.

PANTS_MY_SUBSYSTEM_ENV
  env
      default: my-default
      current value: namespace(__CFBundleI ..... continues with a full dump of my OS process environment)

This is the result when you have a flag configured like:

env = StrOption(….)

or

my_env = StrOption(flag_name="--env" …)

Pants version

2.14.0

OS Are you encountering the bug on MacOS, Linux, or both? MacOS - i don't know if this is an OS specific bug or not. Additional info This is a backfill issue for a request that was already answered via slack. pantsbuild/pants