#21096 Mainline pants has a slow development cycle Issue created by sureshjoshi Pulled from the associated slack chat. Two problems I run into are: • As soon as I pull a new repo,

coursier

runs and starts downloading a bunch of stuff - even though I've never touched our java/jvm/scala/kotlin code • This is made worse, as I have network blocking on - and courier doesn't timeout - just tries to spin forever • Slow cycle time for negligible changes As an example of the 2nd. After fixing a typo in a function comment, and formatting, I spend 31 seconds waiting for that to finish.

time pants fmt src/python/pants/goal/migrate_call_by_name*  
0.58s user 0.06s system 2% cpu 30.932 total

In some more egregious cases, I've waiting up to 60 seconds for a fmt/lint to run, and most of the time happens in the "scheduler" (handwaving everything before running the goal as "scheduler"). I narrowed the bulk of the time spent to commenting out

#"internal_plugins.test_lockfile_fixtures",

in the

pants.toml

- and then went further with removing the JVM entirely. The following diff brought my cycle time down to closer to 6-7 seconds (which I think can be even better). My integration tests also run about 1 second faster, for the 1 test I run. My git diff ``` diff --git a/pants.toml b/pants.toml index 6b91bb1bbd..e52c003510 100644 --- a/pants.toml +++ b/pants.toml @@ -7,7 +7,6 @@ backend_packages.add = [ "pants.backend.build_files.fix.deprecations", "pants.backend.build_files.fmt.black", "pants.backend.python", - "pants.backend.experimental.python.packaging.pyoxidizer", "pants.backend.python.lint.autoflake", "pants.backend.python.lint.black", "pants.backend.python.lint.docformatter", @@ -19,28 +18,28 @@ backend_packages.add = [ "pants.backend.shell", "pants.backend.shell.lint.shellcheck", "pants.backend.shell.lint.shfmt", - "pants.backend.docker", - "pants.backend.docker.lint.hadolint", - "pants.backend.experimental.adhoc", - "pants.backend.experimental.go", - "pants.backend.experimental.java", - "pants.backend.experimental.java.lint.google_java_format", - "pants.backend.experimental.java.debug_goals", + #"pants.backend.docker", + #"pants.backend.docker.lint.hadolint", + #"pants.backend.experimental.adhoc", + #"pants.backend.experimental.go", + #"pants.backend.experimental.java", + #"pants.backend.experimental.java.lint.google_java_format", + #"pants.backend.experimental.java.debug_goals", "pants.backend.experimental.javascript", "pants.backend.experimental.javascript.lint.prettier", "pants.backend.experimental.python", - "pants.backend.experimental.python.packaging.pyoxidizer", - "pants.backend.experimental.scala", - "pants.backend.experimental.scala.lint.scalafmt", - "pants.backend.experimental.scala.lint.scalafix", - "pants.backend.experimental.scala.debug_goals", - "pants.backend.experimental.tools.workunit_logger", - "pants.backend.experimental.visibility", - "pants.backend.tools.preamble", - "pants.backend.tools.taplo", - "pants_explorer.server", - "internal_plugins.releases", - "internal_plugins.test_lockfile_fixtures", + #"pants.backend.experimental.python.packaging.pyoxidizer", + #"pants.backend.experimental.scala", + #"pants.backend.experimental.scala.lint.scalafmt", + #"pants.backend.experimental.scala.lint.scalafix", + #"pants.backend.experimental.scala.debug_goals", + #"pants.backend.experimental.tools.workunit_logger", + #"pants.backend.experimental.visibility", + #"pants.backend.tools.preamble", + #"pants.backend.tools.taplo", + #"pants_explorer.server", + #"internal_plugins.releases", + #"internal_plugins.test_lockfile_fixtures", ] plugins = [ "hdrhistogram", # For use with

--stats-log

. @@ -76,6 +75,12 @@ pants_ignore.add = [ build_ignore.add = [ # Disable Go targets by default so Pants developers do not need Go installed. "testprojects/src/go/**", + "testprojects/src/js/**", + "testprojects/src/jvm/**", + # Why is the docker backend under python? + "testprojects/src/python/docker/**", + "testprojects/src/ts/**", + "3rdparty/jvm/**", ] unmatched_build_file_globs = "error" @@ -203,8 +208,8 @@ requirements = ["//3rdparty/python:mypy"] [coverage-py] interpreter_constraints = ["==3.9.*"] -[preamble] -template_by_globs = "@build-support/preambles/config.yaml" +#[preamble] +#template_by_globs = "@build-support/preambles/config.yaml" [generate-lockfiles] diff = true @@ -213,30 +218,30 @@ diff = true args = ["--keep-runtime-typing", "--py38-plus"] -[jvm] -default_resolve = "jvm_testprojects" +#[jvm] +#default_resolve = "jvm_testprojects" -[jvm.resolves] +#[jvm.resolves] # A shared resolve for all testproject/example code. Because this is not shipped with Pants # binaries, it requires no isolation. -jvm_testprojects = "3rdparty/jvm/testprojects.lockfile" +#jvm_testprojects = "3rdparty/jvm/testprojects.lockfile" # A resolve for the java_parser, which is shipped with Pants, and invoked with its own isolated # classpath. Consequently, we isolate it to its own lockfile. # Note: The jvm_artifact targets in this resolve must be kept in sync with the requirements # in

generate_java_parser_lockfile_request

. -java_parser_dev = "src/python/pants/backend/java/dependency_inference/java_parser.lock" +#java_parser_dev = "src/python/pants/backend/java/dependency_inference/java_parser.lock" # Has the same isolation requirements as

java_parser

. # Note: The jvm_artifact targets in this resolve must be kept in sync with with the requirements # in

generate_scala_parser_lockfile_request

. -scala_parser_dev = "src/python/pants/backend/scala/dependency_inference/scala_parser.lock" -strip_jar_dev = "src/python/pants/jvm/strip_jar/strip_jar.lock" -jar_tool_dev = "src/python/pants/jvm/jar_tool/jar_tool.lock" +#scala_parser_dev = "src/python/pants/backend/scala/dependency_inference/scala_parser.lock" +#strip_jar_dev = "src/python/pants/jvm/strip_jar/strip_jar.lock" +#jar_tool_dev = "src/python/pants/jvm/jar_tool/jar_tool.lock" -[scala] -version_for_resolve = { "scala_parser_dev" = "2.13.8" } +#[scala] +#version_for_resolve = { "scala_parser_dev" = "2.13.8" } -[scalac] -args = ["-Yrangepos", "-Xlint:unused"] +#[scalac] +#args = ["-Yrangepos", "-Xlint:unused"] -[scala-infer] -force_add_siblings_as_dependencies = false +#[scala-infer] +#force_add_siblings_as_dependencies = false diff --git a/src/python/pants/backend/java/dependency_inference/BUILD b/src/python/pants/backend/java/dependency_inference/BUILD index d1d51d5dd1..f4d671b9e6 100644 --- a/src/python/pants/backend/java/dependency_inference/BUILD +++ b/src/python/pants/backend/java/dependency_inference/BUILD @@ -5,8 +5,8 @@ python_sources(dependencies=[":java_resources"]) resources(name="java_resources", sources=["*.java", "java_parser.lock"]) python_tests(name="tests", timeout=240) -# Targets for developing on the Java parser outside of engine rules. -java_sources( - name="java_parser", - resolve="java_parser_dev", -) +# # Targets for developing on the Java parser outside of engine rules. +# java_sources( +# name="java_parser", +# resolve="java_parser_dev", +# ) diff --git a/src/python/pants/backend/scala/dependency_inference/BUILD b/src/python/pants/backend/scala/dependency_inference/BUILD index 26b58ac2d7..2ed0e200c1 100644 --- a/src/python/pants/backend/scala/dependency_inference/BUILD +++ b/src/python/pants/backend/scala/dependency_inference/BUILD @@ -7,18 +7,18 @@ resources(name="scala_resources", sources=["*.scala", "scala_parser.lock"]) python_tests(name="tests", timeout=240) # Targets for developing on the Scala parser outside of engine rules. -scala_sources( - name="scala_parser", - resolve="scala_parser_dev", - # TODO: Allow the parser files to be formatted and linted. - skip_scalafmt=True, - skip_scalafix=True, -) +# scala_sources( +# name="scala_parser", +# resolve="scala_parser_dev", +# # TODO: Allow the parser files to be formatt… pantsbuild/pants