Skip to content

Commit

Permalink
Merge pull request ydb-platform#11021 from ydb-platform/mergelibs-241…
Browse files Browse the repository at this point in the history
…028-2033

Library import 241028-2033
  • Loading branch information
maximyurchuk authored Oct 29, 2024
2 parents 06b8cb8 + fe7b3aa commit c3167d3
Show file tree
Hide file tree
Showing 395 changed files with 13,752 additions and 10,120 deletions.
3 changes: 2 additions & 1 deletion .piglet-meta.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
{
"project":"ydblib"
"project":"ydblib",
"repository":"arcadia"
}
1 change: 0 additions & 1 deletion build/conf/autoincludes.json

This file was deleted.

1 change: 1 addition & 0 deletions build/conf/fbs.conf
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ multimodule FBS_LIBRARY {
module PY2_FBS: PY2_LIBRARY {
.IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER
.ALIASES=SRCS=PY_SRCS
.INCLUDE_TAG=no
ENABLE(PY2_FBS)
SET(PEERDIR_TAGS PY2 PY2_FBS)
# Can not use NO_LINT(), because is not allowed outside of contrib directory
Expand Down
2 changes: 1 addition & 1 deletion build/conf/python.conf
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ when ($BUILD_PYTHON3_BIN) {
YMAKE_PYTHON3_PEERDIR=
}
otherwise {
YMAKE_PYTHON3=${YMAKE_PYTHON3_RESOURCE_GLOBAL}/python3
YMAKE_PYTHON3=${YMAKE_PYTHON3_RESOURCE_GLOBAL}/bin/python3
}

# tag:python-specific
Expand Down
2 changes: 0 additions & 2 deletions build/conf/settings.conf
Original file line number Diff line number Diff line change
Expand Up @@ -125,5 +125,3 @@ _FOLDABLE_VARS=\

ARCADIA_TEST_ROOT=../arcadia_tests_data/
DEFAULT_REQUIREMENTS=network:restricted cpu:1 ram:32

AUTOINCLUDE_PATHS=build/conf/autoincludes.json
10 changes: 5 additions & 5 deletions build/conf/ts/ts_test.conf
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ module TS_TEST_JEST_FOR: _TS_TEST_BASE {
.CMD=TS_TEST_JEST_CMD

# for multimodule peers we should choose NODE_MODULES
SET(PEERDIR_TAGS TS)
SET(PEERDIR_TAGS TS TS_PROTO)

# compatibility with old TS_TEST_SRCS
SET(TS_TEST_EXTENSION test.(ts|tsx|js|jsx))
Expand All @@ -46,7 +46,7 @@ module TS_TEST_JEST_FOR: _TS_TEST_BASE {
_TS_ADD_NODE_MODULES_FOR_BUILDER()
}

TS_TEST_HERMIONE_CMD=$TOUCH_UNIT \
TS_TEST_HERMIONE_CMD=$TOUCH_UNIT ${hide:PEERS} \
&& ${cwd:BINDIR} $MOVE_FILE ${input:TS_TEST_NM} ${output:"workspace_node_modules.tar"} \
${hide;kv:"p TSHRM"} ${hide;kv:"pc magenta"}

Expand All @@ -67,7 +67,7 @@ module TS_TEST_HERMIONE_FOR: _TS_TEST_BASE {
.CMD=TS_TEST_HERMIONE_CMD

# for multimodule peers we should choose TS
SET(PEERDIR_TAGS TS)
SET(PEERDIR_TAGS TS TS_PROTO)

# compatibility with old TS_TEST_SRCS
SET(TS_TEST_EXTENSION hermione.(ts|js))
Expand Down Expand Up @@ -101,7 +101,7 @@ module TS_TEST_PLAYWRIGHT_FOR: _TS_TEST_BASE {
.CMD=TS_TEST_PLAYWRIGHT_CMD

# for multimodule peers we should choose TS
SET(PEERDIR_TAGS TS)
SET(PEERDIR_TAGS TS TS_PROTO)

# compatibility with old TS_TEST_SRCS
SET(TS_TEST_EXTENSION (playwright|spec).(ts|js))
Expand Down Expand Up @@ -134,7 +134,7 @@ module TS_TEST_PLAYWRIGHT_LARGE_FOR: _TS_TEST_BASE {
.CMD=TS_TEST_PLAYWRIGHT_LARGE_CMD

# for multimodule peers we should choose TS
SET(PEERDIR_TAGS TS)
SET(PEERDIR_TAGS TS TS_PROTO)

# compatibility with old TS_TEST_SRCS
SET(TS_TEST_EXTENSION (playwright|spec).(ts|js))
Expand Down
42 changes: 38 additions & 4 deletions build/mapping.conf.json
Original file line number Diff line number Diff line change
Expand Up @@ -418,8 +418,12 @@
"7258655189": "https://devtools-registry.s3.yandex.net/7258655189",
"7268412397": "https://devtools-registry.s3.yandex.net/7268412397",
"7268415538": "https://devtools-registry.s3.yandex.net/7268415538",
"7303146160": "https://devtools-registry.s3.yandex.net/7303146160",
"7303155140": "https://devtools-registry.s3.yandex.net/7303155140",
"7312971312": "https://devtools-registry.s3.yandex.net/7312971312",
"7313006036": "https://devtools-registry.s3.yandex.net/7313006036",
"7328789439": "https://devtools-registry.s3.yandex.net/7328789439",
"7328798659": "https://devtools-registry.s3.yandex.net/7328798659",
"7347465174": "https://devtools-registry.s3.yandex.net/7347465174",
"7347467510": "https://devtools-registry.s3.yandex.net/7347467510",
"5486731632": "https://devtools-registry.s3.yandex.net/5486731632",
"5514350352": "https://devtools-registry.s3.yandex.net/5514350352",
"5514360398": "https://devtools-registry.s3.yandex.net/5514360398",
Expand Down Expand Up @@ -844,8 +848,11 @@
"6990868751": "https://devtools-registry.s3.yandex.net/6990868751",
"6990860705": "https://devtools-registry.s3.yandex.net/6990860705",
"6990881789": "https://devtools-registry.s3.yandex.net/6990881789",
"7324461836": "https://devtools-registry.s3.yandex.net/7324461836",
"7193803465": "https://devtools-registry.s3.yandex.net/7193803465",
"7324464594": "https://devtools-registry.s3.yandex.net/7324464594",
"7193800506": "https://devtools-registry.s3.yandex.net/7193800506",
"7324461714": "https://devtools-registry.s3.yandex.net/7324461714",
"7193813071": "https://devtools-registry.s3.yandex.net/7193813071",
"3167009386": "https://devtools-registry.s3.yandex.net/3167009386",
"3050798466": "https://devtools-registry.s3.yandex.net/3050798466",
Expand Down Expand Up @@ -938,6 +945,16 @@
"5647712429": "https://devtools-registry.s3.yandex.net/5647712429",
"5690801745": "https://devtools-registry.s3.yandex.net/5690801745",
"5731299437": "https://devtools-registry.s3.yandex.net/5731299437",
"7320178938": "https://devtools-registry.s3.yandex.net/7320178938",
"7335838685": "https://devtools-registry.s3.yandex.net/7335838685",
"7320191969": "https://devtools-registry.s3.yandex.net/7320191969",
"7335977723": "https://devtools-registry.s3.yandex.net/7335977723",
"7320114121": "https://devtools-registry.s3.yandex.net/7320114121",
"7335939646": "https://devtools-registry.s3.yandex.net/7335939646",
"7320108874": "https://devtools-registry.s3.yandex.net/7320108874",
"7336031045": "https://devtools-registry.s3.yandex.net/7336031045",
"7320257970": "https://devtools-registry.s3.yandex.net/7320257970",
"7336062757": "https://devtools-registry.s3.yandex.net/7336062757",
"6048579718": "https://devtools-registry.s3.yandex.net/6048579718",
"2980468199": "https://devtools-registry.s3.yandex.net/2980468199",
"5562224408": "https://devtools-registry.s3.yandex.net/5562224408"
Expand Down Expand Up @@ -1360,8 +1377,12 @@
"7258655189": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7268412397": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7268415538": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7303146160": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7303155140": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7312971312": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7313006036": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7328789439": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7328798659": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7347465174": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"7347467510": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
"5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
"5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
"5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
Expand Down Expand Up @@ -1786,8 +1807,11 @@
"6990868751": "none-none-none-sandbox/backup/3527d100-e2d0-4b0e-bb7a-905010853d98/yfm-docs.tar",
"6990860705": "none-none-none-sandbox/backup/d386643e-58f8-43e1-8760-341d73801df8/yfm-docs.tar",
"6990881789": "none-none-none-sandbox/backup/efc428e5-52a5-4a6f-8f0c-53f1d255efea/yfm-docs.tar",
"7324461836": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/0541e185-8261-4b07-9149-257f03a9c8ae/yfm-docs.tar",
"7193803465": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/17df2ad2-24bc-49e8-8909-b58685dac393/yfm-docs.tar",
"7324464594": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/32cc8c74-decd-44a8-bc8c-f8f0d7edfffe/yfm-docs.tar",
"7193800506": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/9be8ed55-d7f8-4029-a7fd-fbfa072b896f/yfm-docs.tar",
"7324461714": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/b3543418-58d4-4e1c-b2be-43b55b035e91/yfm-docs.tar",
"7193813071": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/b6531a79-b803-4672-a9e9-f9f348009f5f/yfm-docs.tar",
"3167009386": "openjdk 11.0.15 vanilla for darwin",
"3050798466": "openjdk 11.0.15 vanilla for darwin-arm64",
Expand Down Expand Up @@ -1880,6 +1904,16 @@
"5647712429": "ymake.exe for win32-clang-cl",
"5690801745": "ymake.exe for win32-clang-cl",
"5731299437": "ymake.exe for win32-clang-cl",
"7320178938": "ynd-bin-frozen-python3-darwin-arm64-e139f4d2038ecfbb34fbbd77f84c4b63ddcb613b",
"7335838685": "ynd-bin-frozen-python3-darwin-arm64-eacbfe8057414a218b407c837c9dcd8f2162f810",
"7320191969": "ynd-bin-frozen-python3-darwin-x86_64-e139f4d2038ecfbb34fbbd77f84c4b63ddcb613b",
"7335977723": "ynd-bin-frozen-python3-darwin-x86_64-eacbfe8057414a218b407c837c9dcd8f2162f810",
"7320114121": "ynd-bin-frozen-python3-linux-aarch64-e139f4d2038ecfbb34fbbd77f84c4b63ddcb613b",
"7335939646": "ynd-bin-frozen-python3-linux-aarch64-eacbfe8057414a218b407c837c9dcd8f2162f810",
"7320108874": "ynd-bin-frozen-python3-linux-x86_64-e139f4d2038ecfbb34fbbd77f84c4b63ddcb613b",
"7336031045": "ynd-bin-frozen-python3-linux-x86_64-eacbfe8057414a218b407c837c9dcd8f2162f810",
"7320257970": "ynd-bin-frozen-python3-mingw-w64-x86_64-e139f4d2038ecfbb34fbbd77f84c4b63ddcb613b",
"7336062757": "ynd-bin-frozen-python3-mingw-w64-x86_64-eacbfe8057414a218b407c837c9dcd8f2162f810",
"6048579718": "yt/go/ytrecipe/cmd/ytexec for linux",
"2980468199": "ytexec for linux",
"5562224408": "ytexec for linux"
Expand Down
16 changes: 8 additions & 8 deletions build/platform/python/ymake_python3/resources.json
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
{
"by_platform": {
"darwin-x86_64": {
"uri": "sbr:7203671634"
},
"darwin-arm64": {
"uri": "sbr:7203671142"
"uri": "sbr:7335838685"
},
"linux-x86_64": {
"uri": "sbr:7203672677"
"darwin-x86_64": {
"uri": "sbr:7335977723"
},
"linux-aarch64": {
"uri": "sbr:7203672172"
"uri": "sbr:7335939646"
},
"linux-x86_64": {
"uri": "sbr:7336031045"
},
"win32-x86_64": {
"uri": "sbr:7203670792"
"uri": "sbr:7336062757"
}
}
}
10 changes: 5 additions & 5 deletions build/platform/test_tool/host.ya.make.inc
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303154871)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347466790)
ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303154594)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347466501)
ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303155140)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347467510)
ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303154435)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347466127)
ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303155023)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347467244)

ENDIF()
10 changes: 5 additions & 5 deletions build/platform/test_tool/host_os.ya.make.inc
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303145767)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347464693)
ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303145350)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347464406)
ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303146160)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347465174)
ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303145126)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347464141)
ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7303145973)
DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7347464906)

ENDIF()
8 changes: 4 additions & 4 deletions build/platform/yfm/ya.make
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ ENDIF()

DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE(
YFM_TOOL
sbr:7193813071 FOR DARWIN-ARM64
sbr:7193813071 FOR DARWIN
sbr:7193800506 FOR LINUX
sbr:7193803465 FOR WIN32
sbr:7324461714 FOR DARWIN-ARM64
sbr:7324461714 FOR DARWIN
sbr:7324464594 FOR LINUX
sbr:7324461836 FOR WIN32
)

END()
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def calc_prepare_deps_inouts_and_resources(
pass

@abstractmethod
def calc_node_modules_inouts(self, local_cli=False) -> tuple[list[str], list[str]]:
def calc_node_modules_inouts(self, local_cli: bool, has_deps: bool) -> tuple[list[str], list[str]]:
pass

@abstractmethod
Expand Down
7 changes: 7 additions & 0 deletions build/plugins/lib/nots/package_manager/base/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,13 @@ def build_nm_bundle_path(p):
return os.path.join(p, NODE_MODULES_WORKSPACE_BUNDLE_FILENAME)


def build_nm_store_path(moddir: str) -> str:
nots_store_path = os.getenv("NOTS_STORE_PATH", os.path.join(home_dir(), ".nots"))
build_path = os.path.join(nots_store_path, "nm_store", moddir)

return build_path


def extract_package_name_from_path(p):
# if we have scope prefix then we are using the first two tokens, otherwise - only the first one
parts = p.split("/", 2)
Expand Down
16 changes: 3 additions & 13 deletions build/plugins/lib/nots/package_manager/npm/npm_package_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,38 +64,28 @@ def calc_prepare_deps_inouts_and_resources(
resources = []

if has_deps:
for dep_path in self.get_local_peers_from_package_json():
ins.append(b_rooted(build_ws_config_path(dep_path)))

for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]):
resources.append(pkg.to_uri())
outs.append(b_rooted(self._tarballs_store_path(pkg, store_path)))

return ins, outs, resources

def calc_node_modules_inouts(self, local_cli=False) -> tuple[list[str], list[str]]:
def calc_node_modules_inouts(self, local_cli: bool, has_deps: bool) -> tuple[list[str], list[str]]:
"""
Returns input and output paths for command that creates `node_modules` bundle.
It relies on .PEERDIRSELF=TS_PREPARE_DEPS
Inputs:
- source package.json
- merged pre-lockfiles and workspace configs of TS_PREPARE_DEPS
Outputs:
- created node_modules bundle
"""
ins = [
s_rooted(build_pj_path(self.module_path)),
b_rooted(build_ws_config_path(self.module_path)),
]
outs = []

pj = self.load_package_json_from_dir(self.sources_path)
if pj.has_dependencies():
ins.append(b_rooted(build_pre_lockfile_path(self.module_path)))
if not local_cli:
outs.append(b_rooted(build_nm_bundle_path(self.module_path)))
for dep_path in self.get_local_peers_from_package_json():
ins.append(b_rooted(build_pj_path(dep_path)))
if not local_cli and has_deps:
outs.append(b_rooted(build_nm_bundle_path(self.module_path)))

return ins, outs

Expand Down
35 changes: 16 additions & 19 deletions build/plugins/lib/nots/package_manager/pnpm/package_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,15 @@
from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
from ..base.node_modules_bundler import bundle_node_modules
from ..base.timeit import timeit
from ..base.utils import b_rooted, build_nm_bundle_path, build_pj_path, home_dir, s_rooted
from ..base.utils import (
b_rooted,
build_nm_bundle_path,
build_nm_path,
build_nm_store_path,
build_pj_path,
home_dir,
s_rooted,
)


class PnpmPackageManager(BasePackageManager):
Expand Down Expand Up @@ -55,8 +63,10 @@ def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bund
if local_cli:
# Use single CAS for all the projects built locally
store_dir = self.get_local_pnpm_store()
# It's a default value of pnpm itself. But it should be defined explicitly for not using values from the lockfiles or from the previous installations.
virtual_store_dir = self._nm_path('.pnpm')

nm_store_path = build_nm_store_path(self.module_path)
# Use single virtual-store location in ~/.nots/nm_store/$MODDIR/node_modules/.pnpm/virtual-store
virtual_store_dir = os.path.join(build_nm_path(nm_store_path), self._VSTORE_NM_PATH)

self._run_pnpm_install(store_dir, virtual_store_dir)
self._run_apply_addons_if_need(yatool_prebuilder_path, virtual_store_dir)
Expand Down Expand Up @@ -106,39 +116,26 @@ def calc_prepare_deps_inouts_and_resources(
resources = []

if has_deps:
for dep_path in self.get_local_peers_from_package_json():
ins.append(b_rooted(build_ws_config_path(dep_path)))
ins.append(b_rooted(build_pre_lockfile_path(dep_path)))

for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]):
resources.append(pkg.to_uri())
outs.append(b_rooted(self._tarballs_store_path(pkg, store_path)))

return ins, outs, resources

# TODO: FBP-1254
# def calc_node_modules_inouts(self, local_cli=False) -> (list[str], list[str]):
def calc_node_modules_inouts(self, local_cli=False):
def calc_node_modules_inouts(self, local_cli: bool, has_deps: bool) -> tuple[list[str], list[str]]:
"""
Returns input and output paths for command that creates `node_modules` bundle.
It relies on .PEERDIRSELF=TS_PREPARE_DEPS
Inputs:
- source package.json
- merged pre-lockfiles and workspace configs of TS_PREPARE_DEPS
Outputs:
- created node_modules bundle
"""
ins = [s_rooted(build_pj_path(self.module_path))]
outs = []

pj = self.load_package_json_from_dir(self.sources_path)
if pj.has_dependencies():
ins.append(b_rooted(build_pre_lockfile_path(self.module_path)))
ins.append(b_rooted(build_ws_config_path(self.module_path)))
if not local_cli:
outs.append(b_rooted(build_nm_bundle_path(self.module_path)))
for dep_path in self.get_local_peers_from_package_json():
ins.append(b_rooted(build_pj_path(dep_path)))
if not local_cli and has_deps:
outs.append(b_rooted(build_nm_bundle_path(self.module_path)))

return ins, outs

Expand Down
4 changes: 2 additions & 2 deletions build/plugins/lib/nots/typescript/ts_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ def filter_files(self, all_files):
return ts_glob(ts_glob_config, all_files)

def get_out_dirs(self):
# type: () -> list[str]
# type: () -> set[str]
output_dirs = [self.compiler_option("outDir"), self.compiler_option("declarationDir")]

return [d for d in output_dirs if d is not None]
return {d for d in output_dirs if d is not None}
Loading

0 comments on commit c3167d3

Please sign in to comment.