From 4a35e0ba401e45659b75a2eea6d9a6e74ac8fff3 Mon Sep 17 00:00:00 2001 From: Felix Wang Date: Sun, 13 Aug 2023 20:23:03 -0700 Subject: [PATCH 01/18] ci: Temporarily disable smoke tests (#3727) Signed-off-by: Felix Wang --- .github/workflows/build_wheels.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index a34be284d5c..6e6539cf9e4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -192,11 +192,12 @@ jobs: echo "$VERSION_OUTPUT from installed wheel is not in the correct format or doesn't have the right version $VERSION." exit 1 fi - - name: Smoke test - run: | - feast init test_repo - cd test_repo/feature_repo - feast apply - echo "$TEST_SCRIPT" > run-and-wait.sh - bash run-and-wait.sh feast serve - bash run-and-wait.sh feast ui + # This is temporarily disabled. + # - name: Smoke test + # run: | + # feast init test_repo + # cd test_repo/feature_repo + # feast apply + # echo "$TEST_SCRIPT" > run-and-wait.sh + # bash run-and-wait.sh feast serve + # bash run-and-wait.sh feast ui From 7bc7c47b4507310850474290131c03fb6d480834 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Sun, 13 Aug 2023 23:33:47 -0400 Subject: [PATCH 02/18] fix: Run store.plan() only when need it. (#3708) fix: issue-3682, run store.plan() only when need it. --- sdk/python/feast/repo_operations.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 0cd425a46be..120f6e7a422 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -296,8 +296,6 @@ def apply_total_with_repo_instance( for data_source in data_sources: data_source.validate(store.config) - registry_diff, infra_diff, new_infra = store.plan(repo) - # For each object in the registry, determine whether it should be kept or deleted. ( all_to_apply, @@ -306,9 +304,10 @@ def apply_total_with_repo_instance( views_to_delete, ) = extract_objects_for_apply_delete(project, registry, repo) - click.echo(registry_diff.to_string()) - if store._should_use_plan(): + registry_diff, infra_diff, new_infra = store.plan(repo) + click.echo(registry_diff.to_string()) + store._apply_diffs(registry_diff, infra_diff, new_infra) click.echo(infra_diff.to_string()) else: From e81684d4f7916c986fa8e6cf06c2918951469799 Mon Sep 17 00:00:00 2001 From: crispin-ki <131684659+crispin-ki@users.noreply.github.com> Date: Wed, 16 Aug 2023 20:42:10 +0100 Subject: [PATCH 03/18] fix: Fix python unit tests (#3734) * Fix regex log_string_to_wait_for to be more generic for changing versions of mysql Signed-off-by: Crispin Logan * Add noqa otherwise flake incorrectly flags W605 Signed-off-by: Crispin Logan --------- Signed-off-by: Crispin Logan --- sdk/python/tests/unit/test_sql_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/unit/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py index 1c2b5a36ddf..39896d3a9dd 100644 --- a/sdk/python/tests/unit/test_sql_registry.py +++ b/sdk/python/tests/unit/test_sql_registry.py @@ -91,7 +91,7 @@ def mysql_registry(): container.start() # The log string uses '8.0.*' since the version might be changed as new Docker images are pushed. - log_string_to_wait_for = "/usr/sbin/mysqld: ready for connections. Version: '8.0.*' socket: '/var/run/mysqld/mysqld.sock' port: 3306" + log_string_to_wait_for = "/usr/sbin/mysqld: ready for connections. Version: '(\d+(\.\d+){1,2})' socket: '/var/run/mysqld/mysqld.sock' port: 3306" # noqa: W605 waited = wait_for_logs( container=container, predicate=log_string_to_wait_for, From 1695c13fa8f48fdc2b5e627837043c5eea0914a9 Mon Sep 17 00:00:00 2001 From: crispin-ki <131684659+crispin-ki@users.noreply.github.com> Date: Mon, 28 Aug 2023 06:26:49 +0100 Subject: [PATCH 04/18] fix: Update py3.8 ci requirements for cython 3.0 release (#3735) * Update py3.8 ci reqs due to cython 3.0 release Signed-off-by: Crispin Logan * Update py3.9 ci reqs due to cython 3.0 release Signed-off-by: Crispin Logan * Update py3.10 ci reqs due to cython 3.0 release Signed-off-by: Crispin Logan * Update py3.8 reqs due to cython 3.0 release Signed-off-by: Crispin Logan * Update py3.9 reqs due to cython 3.0 release Signed-off-by: Crispin Logan * Update py3.10 reqs due to cython 3.0 release Signed-off-by: Crispin Logan --------- Signed-off-by: Crispin Logan --- sdk/python/requirements/py3.10-ci-requirements.txt | 6 +++--- sdk/python/requirements/py3.10-requirements.txt | 4 ++-- sdk/python/requirements/py3.8-ci-requirements.txt | 6 +++--- sdk/python/requirements/py3.8-requirements.txt | 4 ++-- sdk/python/requirements/py3.9-ci-requirements.txt | 6 +++--- sdk/python/requirements/py3.9-requirements.txt | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 6586bdef335..2f28a271dc3 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -206,7 +206,7 @@ executing==1.2.0 # via stack-data fastapi==0.95.2 # via feast (setup.py) -fastavro==1.7.4 +fastavro==1.8.1 # via # feast (setup.py) # pandavro @@ -702,7 +702,7 @@ pyjwt[crypto]==2.7.0 # adal # msal # snowflake-connector-python -pymssql==2.2.7 +pymssql==2.2.8 # via feast (setup.py) pymysql==1.0.3 # via feast (setup.py) @@ -767,7 +767,7 @@ pytz==2023.3 # pandas # snowflake-connector-python # trino -pyyaml==6.0 +pyyaml==6.0.1 # via # dask # feast (setup.py) diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 3eab83fe48c..d9b70d51974 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -43,7 +43,7 @@ exceptiongroup==1.1.1 # via anyio fastapi==0.95.2 # via feast (setup.py) -fastavro==1.7.4 +fastavro==1.8.1 # via # feast (setup.py) # pandavro @@ -133,7 +133,7 @@ python-dotenv==1.0.0 # via uvicorn pytz==2023.3 # via pandas -pyyaml==6.0 +pyyaml==6.0.1 # via # dask # feast (setup.py) diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 85e089b03a9..de814374b68 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -210,7 +210,7 @@ executing==1.2.0 # via stack-data fastapi==0.95.2 # via feast (setup.py) -fastavro==1.7.4 +fastavro==1.8.1 # via # feast (setup.py) # pandavro @@ -713,7 +713,7 @@ pyjwt[crypto]==2.7.0 # adal # msal # snowflake-connector-python -pymssql==2.2.7 +pymssql==2.2.8 # via feast (setup.py) pymysql==1.0.3 # via feast (setup.py) @@ -779,7 +779,7 @@ pytz==2023.3 # pandas # snowflake-connector-python # trino -pyyaml==6.0 +pyyaml==6.0.1 # via # dask # feast (setup.py) diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index b374356b24d..55ee6349048 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -43,7 +43,7 @@ exceptiongroup==1.1.1 # via anyio fastapi==0.95.2 # via feast (setup.py) -fastavro==1.7.4 +fastavro==1.8.1 # via # feast (setup.py) # pandavro @@ -137,7 +137,7 @@ python-dotenv==1.0.0 # via uvicorn pytz==2023.3 # via pandas -pyyaml==6.0 +pyyaml==6.0.1 # via # dask # feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 55a283fb245..aa79b8ec3be 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -206,7 +206,7 @@ executing==1.2.0 # via stack-data fastapi==0.95.2 # via feast (setup.py) -fastavro==1.7.4 +fastavro==1.8.1 # via # feast (setup.py) # pandavro @@ -705,7 +705,7 @@ pyjwt[crypto]==2.7.0 # adal # msal # snowflake-connector-python -pymssql==2.2.7 +pymssql==2.2.8 # via feast (setup.py) pymysql==1.0.3 # via feast (setup.py) @@ -770,7 +770,7 @@ pytz==2023.3 # pandas # snowflake-connector-python # trino -pyyaml==6.0 +pyyaml==6.0.1 # via # dask # feast (setup.py) diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index a96864ef531..53b0ff0e584 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -43,7 +43,7 @@ exceptiongroup==1.1.1 # via anyio fastapi==0.95.2 # via feast (setup.py) -fastavro==1.7.4 +fastavro==1.8.1 # via # feast (setup.py) # pandavro @@ -133,7 +133,7 @@ python-dotenv==1.0.0 # via uvicorn pytz==2023.3 # via pandas -pyyaml==6.0 +pyyaml==6.0.1 # via # dask # feast (setup.py) From 028cc20a28118bd31deca8965782d5ad25f74300 Mon Sep 17 00:00:00 2001 From: Danny C Date: Sun, 27 Aug 2023 22:29:14 -0700 Subject: [PATCH 05/18] fix: Pin protobuf version to avoid seg fault on some machines Signed-off-by: Danny C --- Makefile | 2 +- setup.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a104bf9ca5a..cf8a899ac68 100644 --- a/Makefile +++ b/Makefile @@ -353,7 +353,7 @@ kill-trino-locally: cd ${ROOT_DIR}; docker stop trino install-protoc-dependencies: - pip install --ignore-installed protobuf grpcio-tools==1.47.0 mypy-protobuf==3.1.0 + pip install --ignore-installed protobuf==4.23.4 grpcio-tools==1.47.0 mypy-protobuf==3.1.0 install-feast-ci-locally: pip install -e ".[ci]" diff --git a/setup.py b/setup.py index aee2ed0b3eb..f10322ed8a3 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,8 @@ "pandas>=1.4.3,<2", # For some reason pandavro higher than 1.5.* only support pandas less than 1.3. "pandavro~=1.5.0", - "protobuf<5,>3.20", + # Higher than 4.23.4 seems to cause a seg fault + "protobuf<4.23.4,>3.20", "proto-plus>=1.20.0,<2", "pyarrow>=4,<12", "pydantic>=1,<2", From eb5f9ae0bbaf01d0d83c2bb8d2a9bdb668f8267a Mon Sep 17 00:00:00 2001 From: Danny C Date: Sun, 27 Aug 2023 23:19:38 -0700 Subject: [PATCH 06/18] ci: Pin pip version to avoid test breakage Signed-off-by: Danny C --- .github/workflows/java_master_only.yml | 2 +- .github/workflows/java_pr.yml | 2 +- .github/workflows/linter.yml | 2 +- .github/workflows/master_only.yml | 2 +- .github/workflows/nightly-ci.yml | 2 +- .github/workflows/pr_integration_tests.yml | 2 +- .github/workflows/pr_local_integration_tests.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 00d8e628c97..23a12259cad 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -114,7 +114,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 152ca64cdb1..03aa6d63cd5 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -150,7 +150,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 0f85cddf2dd..f033972917d 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -17,7 +17,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 517095d7ece..4ec109a39ff 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -108,7 +108,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index fd5bc9f85f4..4ecd302c036 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -175,7 +175,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 6b19b2f6300..47def1dcf9e 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -135,7 +135,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 527d279f10a..db99cad63b0 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -40,7 +40,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 37457250223..992a3168c32 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -38,7 +38,7 @@ jobs: ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib - name: Upgrade pip version run: | - pip install --upgrade pip + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | From 58aff346832ebde1695a47cf724da3d65a4a8c53 Mon Sep 17 00:00:00 2001 From: Danny C Date: Sun, 27 Aug 2023 23:25:39 -0700 Subject: [PATCH 07/18] ci: Pin pip version to slightly higher to avoid test breakage Signed-off-by: Danny C --- .github/fork_workflows/fork_pr_integration_tests_aws.yml | 2 +- .github/fork_workflows/fork_pr_integration_tests_gcp.yml | 2 +- .github/fork_workflows/fork_pr_integration_tests_snowflake.yml | 2 +- .github/workflows/java_master_only.yml | 2 +- .github/workflows/java_pr.yml | 2 +- .github/workflows/linter.yml | 2 +- .github/workflows/master_only.yml | 2 +- .github/workflows/nightly-ci.yml | 2 +- .github/workflows/pr_integration_tests.yml | 2 +- .github/workflows/pr_local_integration_tests.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index c354dce970d..ab5130c704e 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -111,7 +111,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<22.3" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 07eb323d567..4b74c0ab096 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -55,7 +55,7 @@ jobs: run: gcloud info - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index a86e9ad7a0a..cef970ab16a 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -46,7 +46,7 @@ jobs: - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 23a12259cad..b15ddddc5d7 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -114,7 +114,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 03aa6d63cd5..a927587d7d3 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -150,7 +150,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index f033972917d..c2112ea303f 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -17,7 +17,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 4ec109a39ff..a7d412fd4ac 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -108,7 +108,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 4ecd302c036..03078c0a788 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -175,7 +175,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 47def1dcf9e..ed02636c993 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -135,7 +135,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index db99cad63b0..1a2c0790d1a 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -40,7 +40,7 @@ jobs: architecture: x64 - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 992a3168c32..538a5819799 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -38,7 +38,7 @@ jobs: ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | From 0b3fa13373ba12e912e8b4b1a0dc5ea18cbe8fc9 Mon Sep 17 00:00:00 2001 From: Danny C Date: Mon, 4 Sep 2023 18:56:20 -0700 Subject: [PATCH 08/18] ci: Fix ci breakage from old pip version being restored from pip cache Signed-off-by: Danny C --- .github/fork_workflows/fork_pr_integration_tests_aws.yml | 6 +++--- .github/fork_workflows/fork_pr_integration_tests_gcp.yml | 6 +++--- .../fork_workflows/fork_pr_integration_tests_snowflake.yml | 7 +++---- .github/workflows/java_master_only.yml | 7 +++---- .github/workflows/java_pr.yml | 6 +++--- .github/workflows/linter.yml | 6 +++--- .github/workflows/master_only.yml | 6 +++--- .github/workflows/nightly-ci.yml | 6 +++--- .github/workflows/pr_integration_tests.yml | 6 +++--- .github/workflows/pr_local_integration_tests.yml | 6 +++--- .github/workflows/unit_tests.yml | 6 +++--- 11 files changed, 33 insertions(+), 35 deletions(-) diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index ab5130c704e..7261833ae6b 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -109,9 +109,6 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<22.3" - name: Get pip cache dir id: pip-cache run: | @@ -126,6 +123,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<22.3" - name: Install pip-tools run: pip install pip-tools - name: Install apache-arrow on ubuntu diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 4b74c0ab096..1a05c068b50 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -53,9 +53,6 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI run: gcloud info - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -70,6 +67,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install apache-arrow on ubuntu diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index cef970ab16a..9327f5c7294 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -43,10 +43,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 - - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -61,6 +57,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install apache-arrow on ubuntu diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index b15ddddc5d7..d82f69dd3cb 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -112,9 +112,6 @@ jobs: with: python-version: 3.8 architecture: x64 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -129,9 +126,11 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - - name: Install Python dependencies run: make install-python-ci-dependencies - uses: actions/cache@v2 diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index a927587d7d3..83c52e7dbfd 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -148,9 +148,6 @@ jobs: with: python-version: 3.8 architecture: x64 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -165,6 +162,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install Python dependencies diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index c2112ea303f..a4a42a11edb 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -15,9 +15,6 @@ jobs: with: python-version: "3.8" architecture: x64 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -32,6 +29,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install dependencies diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index a7d412fd4ac..580ea3171b3 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -106,9 +106,6 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -123,6 +120,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install dependencies diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 03078c0a788..0e1df81262d 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -173,9 +173,6 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -190,6 +187,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install apache-arrow on ubuntu diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index ed02636c993..73344ec2ddd 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -133,9 +133,6 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -150,6 +147,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install dependencies diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 1a2c0790d1a..111a9b51a9c 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -38,9 +38,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -55,6 +52,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install dependencies diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 538a5819799..f03cd33346c 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -36,9 +36,6 @@ jobs: run: | brew install zlib ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<23.2" - name: Get pip cache dir id: pip-cache run: | @@ -53,6 +50,9 @@ jobs: key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} restore-keys: | ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<23.2" - name: Install pip-tools run: pip install pip-tools - name: Install dependencies From d7041f4cce813d349e9016da55d65a65c1ec2355 Mon Sep 17 00:00:00 2001 From: Mark Snidal Date: Tue, 5 Sep 2023 02:36:03 -0400 Subject: [PATCH 09/18] fix: Handle unknown postgres source types gracefully (#3634) Handle more pg types gracefully Signed-off-by: Mark Snidal --- sdk/python/feast/type_map.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index df853462836..52e28907505 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -873,13 +873,26 @@ def feast_value_type_to_pa( def pg_type_code_to_pg_type(code: int) -> str: - return { + """ Map the postgres type code a Feast type string + + Rather than raise an exception on an unknown type, we return the + string representation of the type code. This way rather than raising + an exception on unknown types, Feast will just skip the problem columns. + + Note that json and jsonb are not supported but this shows up in the + log as a warning. Since postgres allows custom types we return an unknown for those cases. + + See: https://jdbc.postgresql.org/documentation/publicapi/index.html?constant-values.html + """ + PG_TYPE_MAP = { 16: "boolean", 17: "bytea", 20: "bigint", 21: "smallint", 23: "integer", 25: "text", + 114: "json", + 199: "json[]", 700: "real", 701: "double precision", 1000: "boolean[]", @@ -905,7 +918,11 @@ def pg_type_code_to_pg_type(code: int) -> str: 1700: "numeric", 2950: "uuid", 2951: "uuid[]", - }[code] + 3802: "jsonb", + 3807: "jsonb[]", + } + + return PG_TYPE_MAP.get(code, "unknown") def pg_type_code_to_arrow(code: int) -> str: From 1a1849cf3c6224f4c332fdf7111baed8c5725c25 Mon Sep 17 00:00:00 2001 From: Danny C Date: Tue, 5 Sep 2023 00:15:59 -0700 Subject: [PATCH 10/18] chore: Fix linting issue Signed-off-by: Danny C --- sdk/python/feast/type_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 52e28907505..d246fb40545 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -873,10 +873,10 @@ def feast_value_type_to_pa( def pg_type_code_to_pg_type(code: int) -> str: - """ Map the postgres type code a Feast type string + """Map the postgres type code a Feast type string Rather than raise an exception on an unknown type, we return the - string representation of the type code. This way rather than raising + string representation of the type code. This way rather than raising an exception on unknown types, Feast will just skip the problem columns. Note that json and jsonb are not supported but this shows up in the From 377758b48f89fb7f1b99856d09d3b383a2c80882 Mon Sep 17 00:00:00 2001 From: Ren Date: Tue, 5 Sep 2023 03:18:55 -0400 Subject: [PATCH 11/18] docs: Add Dragonfly as an online-store option to Feast Readme and docs (#3647) * Add dragonfly to feast Readme and docs Signed-off-by: Danny C * Remove new lines Signed-off-by: Danny C * Remove more newlines Signed-off-by: Danny C * rebase and fix using template generation Signed-off-by: Danny C --------- Signed-off-by: Danny C Co-authored-by: Danny C --- README.md | 1 + docs/reference/online-stores/dragonfly.md | 90 +++++++++++++++++++++++ docs/roadmap.md | 1 + 3 files changed, 92 insertions(+) create mode 100644 docs/reference/online-stores/dragonfly.md diff --git a/README.md b/README.md index 1152aa060e4..f2c9348b1cc 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) + * [x] [Dragonfly](https://docs.feast.dev/reference/online-stores/dragonfly) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) diff --git a/docs/reference/online-stores/dragonfly.md b/docs/reference/online-stores/dragonfly.md new file mode 100644 index 00000000000..bcd814ecc45 --- /dev/null +++ b/docs/reference/online-stores/dragonfly.md @@ -0,0 +1,90 @@ +# Dragonfly online store + +## Description + +[Dragonfly](https://github.com/dragonflydb/dragonfly) is a modern in-memory datastore that implements novel algorithms and data structures on top of a multi-threaded, share-nothing architecture. Thanks to its API compatibility, Dragonfly can act as a drop-in replacement for Redis. Due to Dragonfly's hardware efficiency, you can run a single node instance on a small 8GB instance or scale vertically to large 768GB machines with 64 cores. This greatly reduces infrastructure costs as well as architectural complexity. + +Similar to Redis, Dragonfly can be used as an online feature store for Feast. + +## Using Dragonfly as a drop-in Feast online store instead of Redis + +Make sure you have Python and `pip` installed. + +Install the Feast SDK and CLI + +`pip install feast` + +In order to use Dragonfly as the online store, you'll need to install the redis extra: + +`pip install 'feast[redis]'` + +### 1. Create a feature repository + +Bootstrap a new feature repository: + +``` +feast init feast_dragonfly +cd feast_dragonfly/feature_repo +``` + +Update `feature_repo/feature_store.yaml` with the below contents: + +``` +project: feast_dragonfly +registry: data/registry.db +provider: local +online_store: +type: redis +connection_string: "localhost:6379" +``` + +### 2. Start Dragonfly + +There are several options available to get Dragonfly up and running quickly. We will be using Docker for this tutorial. + +`docker run --network=host --ulimit memlock=-1 docker.dragonflydb.io/dragonflydb/dragonfly` + +### 3. Register feature definitions and deploy your feature store + +`feast apply` + +The `apply` command scans python files in the current directory (`example_repo.py` in this case) for feature view/entity definitions, registers the objects, and deploys infrastructure. +You should see the following output: + +``` +.... +Created entity driver +Created feature view driver_hourly_stats_fresh +Created feature view driver_hourly_stats +Created on demand feature view transformed_conv_rate +Created on demand feature view transformed_conv_rate_fresh +Created feature service driver_activity_v1 +Created feature service driver_activity_v3 +Created feature service driver_activity_v2 +``` + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Redis online store. + +| | Redis | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/roadmap.md b/docs/roadmap.md index e75e58849bf..d5dc88005bf 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -33,6 +33,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) * [x] [Bigtable](https://docs.feast.dev/reference/online-stores/bigtable) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) + * [x] [Dragonfly](https://docs.feast.dev/reference/online-stores/dragonfly) * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) From f2c59885e31f3f238dbd9c13cd1ba168e3233a9d Mon Sep 17 00:00:00 2001 From: crispin-ki <131684659+crispin-ki@users.noreply.github.com> Date: Tue, 5 Sep 2023 18:50:47 +0100 Subject: [PATCH 12/18] fix: Remove unwanted excessive splitting of gcs path, so expected gcs parquet paths are returned from BigQueryRetrievalJob.to_remote_storage() (#3730) Remove unwanted excessive splitting of gcs path Signed-off-by: Crispin Logan --- sdk/python/feast/infra/offline_stores/bigquery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 5dd8f61bced..5913b60f62f 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -577,7 +577,6 @@ def to_remote_storage(self) -> List[str]: else: storage_client = StorageClient(project=self.client.project) bucket, prefix = self._gcs_path[len("gs://") :].split("/", 1) - prefix = prefix.rsplit("/", 1)[0] if prefix.startswith("/"): prefix = prefix[1:] From a3fcd1f369bdf07174b5ecf2a49ca9864cf145d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mehmet=20Tokg=C3=B6z?= <56408993+mehmettokgoz@users.noreply.github.com> Date: Thu, 7 Sep 2023 07:50:17 +0300 Subject: [PATCH 13/18] feat: Implement gRPC server to ingest streaming features (#3687) * Implemented gRPC server for ingesting streaming features. Signed-off-by: mehmettokgoz Signed-off-by: Danny C --- Makefile | 2 +- protos/feast/serving/GrpcServer.proto | 27 ++ sdk/python/feast/cli.py | 31 ++ sdk/python/feast/infra/contrib/grpc_server.py | 95 ++++ .../requirements/py3.10-ci-requirements.txt | 400 +++++++++-------- .../requirements/py3.10-requirements.txt | 105 +++-- .../requirements/py3.8-ci-requirements.txt | 386 +++++++++-------- .../requirements/py3.8-requirements.txt | 109 +++-- .../requirements/py3.9-ci-requirements.txt | 408 +++++++++--------- .../requirements/py3.9-requirements.txt | 109 +++-- setup.py | 23 +- 11 files changed, 999 insertions(+), 696 deletions(-) create mode 100644 protos/feast/serving/GrpcServer.proto create mode 100644 sdk/python/feast/infra/contrib/grpc_server.py diff --git a/Makefile b/Makefile index cf8a899ac68..4b85c0e4483 100644 --- a/Makefile +++ b/Makefile @@ -353,7 +353,7 @@ kill-trino-locally: cd ${ROOT_DIR}; docker stop trino install-protoc-dependencies: - pip install --ignore-installed protobuf==4.23.4 grpcio-tools==1.47.0 mypy-protobuf==3.1.0 + pip install --ignore-installed protobuf==4.23.4 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0 install-feast-ci-locally: pip install -e ".[ci]" diff --git a/protos/feast/serving/GrpcServer.proto b/protos/feast/serving/GrpcServer.proto new file mode 100644 index 00000000000..cd0274c5c75 --- /dev/null +++ b/protos/feast/serving/GrpcServer.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +message PushRequest { + map features = 1; + string stream_feature_view = 2; + bool allow_registry_cache = 3; + string to = 4; +} + +message PushResponse { + bool status = 1; +} + +message WriteToOnlineStoreRequest { + map features = 1; + string feature_view_name = 2; + bool allow_registry_cache = 3; +} + +message WriteToOnlineStoreResponse { + bool status = 1; +} + +service GrpcFeatureServer { + rpc Push (PushRequest) returns (PushResponse) {}; + rpc WriteToOnlineStore (WriteToOnlineStoreRequest) returns (WriteToOnlineStoreResponse); +} \ No newline at end of file diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 229cb992321..8c2c326b595 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -28,6 +28,7 @@ from feast.constants import DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError from feast.feature_view import FeatureView +from feast.infra.contrib.grpc_server import get_grpc_server from feast.on_demand_feature_view import OnDemandFeatureView from feast.repo_config import load_repo_config from feast.repo_operations import ( @@ -689,6 +690,36 @@ def serve_command( ) +@cli.command("listen") +@click.option( + "--address", + "-a", + type=click.STRING, + default="localhost:50051", + show_default=True, + help="Address of the gRPC server", +) +@click.option( + "--max_workers", + "-w", + type=click.INT, + default=10, + show_default=False, + help="The maximum number of threads that can be used to execute the gRPC calls", +) +@click.pass_context +def listen_command( + ctx: click.Context, + address: str, + max_workers: int, +): + """Start a gRPC feature server to ingest streaming features on given address""" + store = create_feature_store(ctx) + server = get_grpc_server(address, store, max_workers) + server.start() + server.wait_for_termination() + + @cli.command("serve_transformations") @click.option( "--port", diff --git a/sdk/python/feast/infra/contrib/grpc_server.py b/sdk/python/feast/infra/contrib/grpc_server.py new file mode 100644 index 00000000000..2017f1095b2 --- /dev/null +++ b/sdk/python/feast/infra/contrib/grpc_server.py @@ -0,0 +1,95 @@ +import logging +from concurrent import futures + +import grpc +import pandas as pd +from grpc_health.v1 import health, health_pb2_grpc + +from feast.data_source import PushMode +from feast.errors import PushSourceNotFoundException +from feast.feature_store import FeatureStore +from feast.protos.feast.serving.GrpcServer_pb2 import ( + PushResponse, + WriteToOnlineStoreResponse, +) +from feast.protos.feast.serving.GrpcServer_pb2_grpc import ( + GrpcFeatureServerServicer, + add_GrpcFeatureServerServicer_to_server, +) + + +def parse(features): + df = {} + for i in features.keys(): + df[i] = [features.get(i)] + return pd.DataFrame.from_dict(df) + + +class GrpcFeatureServer(GrpcFeatureServerServicer): + fs: FeatureStore + + def __init__(self, fs: FeatureStore): + self.fs = fs + super().__init__() + + def Push(self, request, context): + try: + df = parse(request.features) + if request.to == "offline": + to = PushMode.OFFLINE + elif request.to == "online": + to = PushMode.ONLINE + elif request.to == "online_and_offline": + to = PushMode.ONLINE_AND_OFFLINE + else: + raise ValueError( + f"{request.to} is not a supported push format. Please specify one of these ['online', 'offline', " + f"'online_and_offline']." + ) + self.fs.push( + push_source_name=request.push_source_name, + df=df, + allow_registry_cache=request.allow_registry_cache, + to=to, + ) + except PushSourceNotFoundException as e: + logging.exception(str(e)) + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + context.set_details(str(e)) + return PushResponse(status=False) + except Exception as e: + logging.exception(str(e)) + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + return PushResponse(status=False) + return PushResponse(status=True) + + def WriteToOnlineStore(self, request, context): + logging.warning( + "write_to_online_store is deprecated. Please consider using Push instead" + ) + try: + df = parse(request.features) + self.fs.write_to_online_store( + feature_view_name=request.feature_view_name, + df=df, + allow_registry_cache=request.allow_registry_cache, + ) + except Exception as e: + logging.exception(str(e)) + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + return PushResponse(status=False) + return WriteToOnlineStoreResponse(status=True) + + +def get_grpc_server(address: str, fs: FeatureStore, max_workers: int): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers)) + add_GrpcFeatureServerServicer_to_server(GrpcFeatureServer(fs), server) + health_servicer = health.HealthServicer( + experimental_non_blocking=True, + experimental_thread_pool=futures.ThreadPoolExecutor(max_workers=max_workers), + ) + health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server) + server.add_insecure_port(address) + return server diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 2f28a271dc3..cb72fdaa350 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -8,7 +8,7 @@ adal==1.2.7 # via msrestazure adlfs==0.5.9 # via feast (setup.py) -aiohttp==3.8.4 +aiohttp==3.8.5 # via # adlfs # gcsfs @@ -18,7 +18,7 @@ alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations -anyio==3.7.0 +anyio==4.0.0 # via # httpcore # jupyter-server @@ -30,11 +30,8 @@ appnope==0.1.3 # via # ipykernel # ipython -argon2-cffi==21.3.0 - # via - # jupyter-server - # nbclassic - # notebook +argon2-cffi==23.1.0 + # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 @@ -45,9 +42,11 @@ asn1crypto==1.5.1 # snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.2.1 +asttokens==2.4.0 # via stack-data -async-timeout==4.0.2 +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 # via # aiohttp # redis @@ -56,9 +55,10 @@ attrs==23.1.0 # aiohttp # bowler # jsonschema + # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.27.0 +azure-core==1.29.3 # via # adlfs # azure-identity @@ -66,16 +66,18 @@ azure-core==1.27.0 # msrest azure-datalake-store==0.0.53 # via adlfs -azure-identity==1.13.0 +azure-identity==1.14.0 # via # adlfs # feast (setup.py) -azure-storage-blob==12.16.0 +azure-storage-blob==12.17.0 # via # adlfs # feast (setup.py) babel==2.12.1 - # via sphinx + # via + # jupyterlab-server + # sphinx backcall==0.2.0 # via ipython beautifulsoup4==4.12.2 @@ -84,30 +86,30 @@ black==22.12.0 # via feast (setup.py) bleach==6.0.0 # via nbconvert -boto3==1.26.146 +boto3==1.28.42 # via # feast (setup.py) # moto -botocore==1.29.146 +botocore==1.31.42 # via # boto3 # moto # s3transfer bowler==0.9.0 # via feast (setup.py) -build==0.10.0 +build==1.0.3 # via # feast (setup.py) # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.0 +cachecontrol==0.13.1 # via firebase-admin cachetools==5.3.1 # via google-auth -cassandra-driver==3.27.0 +cassandra-driver==3.28.0 # via feast (setup.py) -certifi==2023.5.7 +certifi==2023.7.22 # via # httpcore # httpx @@ -122,14 +124,14 @@ cffi==1.15.1 # azure-datalake-store # cryptography # snowflake-connector-python -cfgv==3.3.1 +cfgv==3.4.0 # via pre-commit -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests # snowflake-connector-python -click==8.1.3 +click==8.1.7 # via # black # bowler @@ -146,16 +148,17 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.1.3 - # via ipykernel -coverage[toml]==7.2.7 +comm==0.1.4 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.3.1 # via pytest-cov -cryptography==40.0.2 +cryptography==41.0.3 # via # adal # azure-identity # azure-storage-blob - # cassandra-driver # feast (setup.py) # great-expectations # moto @@ -165,11 +168,11 @@ cryptography==40.0.2 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.5.1 +dask==2023.9.1 # via feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery -debugpy==1.6.7 +debugpy==1.6.7.post1 # via ipykernel decorator==5.1.1 # via @@ -181,12 +184,12 @@ deprecated==1.2.14 # via redis deprecation==2.1.0 # via testcontainers -dill==0.3.6 +dill==0.3.7 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.7 # via virtualenv docker==6.1.3 # via @@ -196,23 +199,24 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via # anyio + # ipython # pytest -execnet==1.9.0 +execnet==2.0.2 # via pytest-xdist executing==1.2.0 # via stack-data -fastapi==0.95.2 +fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.1 +fastavro==1.8.2 # via # feast (setup.py) # pandavro -fastjsonschema==2.17.1 +fastjsonschema==2.18.0 # via nbformat -filelock==3.12.0 +filelock==3.12.3 # via # snowflake-connector-python # virtualenv @@ -224,7 +228,7 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal @@ -239,7 +243,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.0 +google-api-core[grpc]==2.11.1 # via # feast (setup.py) # firebase-admin @@ -251,9 +255,9 @@ google-api-core[grpc]==2.11.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.88.0 +google-api-python-client==2.98.0 # via firebase-admin -google-auth==2.19.1 +google-auth==2.22.0 # via # gcsfs # google-api-core @@ -267,35 +271,35 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-bigquery[pandas]==3.11.0 +google-cloud-bigquery[pandas]==3.11.4 # via feast (setup.py) -google-cloud-bigquery-storage==2.20.0 +google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) -google-cloud-bigtable==2.18.1 +google-cloud-bigtable==2.21.0 # via feast (setup.py) -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.15.2 +google-cloud-datastore==2.18.0 # via feast (setup.py) google-cloud-firestore==2.11.1 # via firebase-admin -google-cloud-storage==2.9.0 +google-cloud-storage==2.10.0 # via # feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.5.0 +google-resumable-media==2.6.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.59.0 +googleapis-common-protos[grpc]==1.60.0 # via # feast (setup.py) # google-api-core @@ -303,30 +307,31 @@ googleapis-common-protos[grpc]==1.59.0 # grpcio-status great-expectations==0.15.50 # via feast (setup.py) -greenlet==2.0.2 - # via sqlalchemy grpc-google-iam-v1==0.12.6 # via google-cloud-bigtable -grpcio==1.54.2 +grpcio==1.57.0 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos # grpc-google-iam-v1 + # grpcio-health-checking # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.54.2 +grpcio-health-checking==1.57.0 # via feast (setup.py) -grpcio-status==1.54.2 +grpcio-reflection==1.57.0 + # via feast (setup.py) +grpcio-status==1.57.0 # via google-api-core -grpcio-testing==1.54.2 +grpcio-testing==1.57.0 # via feast (setup.py) -grpcio-tools==1.54.2 +grpcio-tools==1.57.0 # via feast (setup.py) -gunicorn==20.1.0 +gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 # via @@ -334,21 +339,21 @@ h11==0.14.0 # uvicorn happybase==1.2.0 # via feast (setup.py) -hazelcast-python-client==5.2.0 +hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.2.3 # via feast (setup.py) -httpcore==0.17.2 +httpcore==0.17.3 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 # via feast (setup.py) -identify==2.5.24 +identify==2.5.27 # via pre-commit idna==3.4 # via @@ -360,27 +365,20 @@ idna==3.4 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via # dask # great-expectations iniconfig==2.0.0 # via pytest -ipykernel==6.23.1 - # via - # ipywidgets - # nbclassic - # notebook -ipython==8.14.0 +ipykernel==6.25.2 + # via jupyterlab +ipython==8.15.0 # via # great-expectations # ipykernel # ipywidgets -ipython-genutils==0.2.0 - # via - # nbclassic - # notebook -ipywidgets==8.0.6 +ipywidgets==8.1.0 # via great-expectations isodate==0.6.1 # via @@ -390,7 +388,7 @@ isoduration==20.11.0 # via jsonschema isort==5.12.0 # via feast (setup.py) -jedi==0.18.2 +jedi==0.19.0 # via ipython jinja2==3.1.2 # via @@ -398,56 +396,69 @@ jinja2==3.1.2 # feast (setup.py) # great-expectations # jupyter-server + # jupyterlab + # jupyterlab-server # moto - # nbclassic # nbconvert - # notebook # sphinx jmespath==1.0.1 # via # boto3 # botocore -jsonpatch==1.32 +json5==0.9.14 + # via jupyterlab-server +jsonpatch==1.33 # via great-expectations -jsonpointer==2.3 +jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.17.3 +jsonschema[format-nongpl]==4.19.0 # via # altair # feast (setup.py) # great-expectations # jupyter-events + # jupyterlab-server # nbformat -jupyter-client==8.2.0 +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==8.3.1 # via # ipykernel # jupyter-server - # nbclassic # nbclient - # notebook -jupyter-core==5.3.0 +jupyter-core==5.3.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # nbclient # nbconvert # nbformat - # notebook -jupyter-events==0.6.3 +jupyter-events==0.7.0 # via jupyter-server -jupyter-server==2.6.0 +jupyter-lsp==2.2.0 + # via jupyterlab +jupyter-server==2.7.3 # via - # nbclassic + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server +jupyterlab==4.0.5 + # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-widgets==3.0.7 +jupyterlab-server==2.24.0 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.8 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -460,7 +471,7 @@ markupsafe==2.1.3 # jinja2 # nbconvert # werkzeug -marshmallow==3.19.0 +marshmallow==3.20.1 # via great-expectations matplotlib-inline==0.1.6 # via @@ -470,19 +481,19 @@ mccabe==0.7.0 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.5 +mistune==3.0.1 # via # great-expectations # nbconvert -mmh3==4.0.0 +mmh3==4.0.1 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.1.10 +moto==4.2.2 # via feast (setup.py) -msal==1.22.0 +msal==1.23.0 # via # azure-datalake-store # azure-identity @@ -499,7 +510,7 @@ multidict==6.0.4 # via # aiohttp # yarl -multiprocess==0.70.14 +multiprocess==0.70.15 # via bytewax mypy==0.982 # via @@ -511,37 +522,29 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.1 # via feast (setup.py) -mysqlclient==2.1.1 +mysqlclient==2.2.0 # via feast (setup.py) -nbclassic==1.0.0 - # via notebook nbclient==0.8.0 # via nbconvert -nbconvert==7.4.0 - # via - # jupyter-server - # nbclassic - # notebook -nbformat==5.9.0 +nbconvert==7.8.0 + # via jupyter-server +nbformat==5.9.2 # via # great-expectations # jupyter-server - # nbclassic # nbclient # nbconvert - # notebook -nest-asyncio==1.5.6 - # via - # ipykernel - # nbclassic - # notebook +nest-asyncio==1.5.7 + # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==6.5.4 +notebook==7.0.3 # via great-expectations notebook-shim==0.2.3 - # via nbclassic -numpy==1.24.3 + # via + # jupyterlab + # notebook +numpy==1.25.2 # via # altair # db-dtypes @@ -555,7 +558,7 @@ oauthlib==3.2.2 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python -overrides==7.3.1 +overrides==7.4.0 # via jupyter-server packaging==23.1 # via @@ -566,8 +569,11 @@ packaging==23.1 # docker # google-cloud-bigquery # great-expectations + # gunicorn # ipykernel # jupyter-server + # jupyterlab + # jupyterlab-server # marshmallow # nbconvert # pytest @@ -591,7 +597,7 @@ parso==0.8.3 # via jedi partd==1.4.0 # via dask -pathspec==0.11.1 +pathspec==0.11.2 # via black pbr==5.11.1 # via mock @@ -599,29 +605,27 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.13.0 +pip-tools==7.3.0 # via feast (setup.py) -platformdirs==3.5.1 +platformdirs==3.8.1 # via # black # jupyter-core + # snowflake-connector-python # virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via pytest ply==3.11 # via thriftpy2 portalocker==2.7.0 # via msal-extensions -pre-commit==3.3.2 +pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.17.0 - # via - # jupyter-server - # nbclassic - # notebook -prompt-toolkit==3.0.38 +prometheus-client==0.17.1 + # via jupyter-server +prompt-toolkit==3.0.39 # via ipython -proto-plus==1.22.2 +proto-plus==1.22.3 # via # feast (setup.py) # google-cloud-bigquery @@ -629,7 +633,7 @@ proto-plus==1.22.2 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.2 +protobuf==4.23.3 # via # feast (setup.py) # google-api-core @@ -640,6 +644,7 @@ protobuf==4.23.2 # google-cloud-firestore # googleapis-common-protos # grpc-google-iam-v1 + # grpcio-health-checking # grpcio-reflection # grpcio-status # grpcio-testing @@ -650,7 +655,7 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg2-binary==2.9.6 +psycopg2-binary==2.9.7 # via feast (setup.py) ptyprocess==0.7.0 # via @@ -684,43 +689,41 @@ pycparser==2.21 # via cffi pycryptodomex==3.18.0 # via snowflake-connector-python -pydantic==1.10.8 +pydantic==1.10.12 # via # fastapi # feast (setup.py) # great-expectations pyflakes==3.0.1 # via flake8 -pygments==2.15.1 +pygments==2.16.1 # via # feast (setup.py) # ipython # nbconvert # sphinx -pyjwt[crypto]==2.7.0 +pyjwt[crypto]==2.8.0 # via # adal # msal # snowflake-connector-python pymssql==2.2.8 # via feast (setup.py) -pymysql==1.0.3 +pymysql==1.1.0 # via feast (setup.py) pyodbc==4.0.39 # via feast (setup.py) pyopenssl==23.2.0 # via snowflake-connector-python -pyparsing==3.0.9 +pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 # via build -pyrsistent==0.19.3 - # via jsonschema -pyspark==3.4.0 +pyspark==3.4.1 # via feast (setup.py) -pytest==7.3.1 +pytest==7.4.1 # via # feast (setup.py) # pytest-benchmark @@ -761,7 +764,7 @@ python-dotenv==1.0.0 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3 +pytz==2023.3.post1 # via # great-expectations # pandas @@ -776,16 +779,19 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.0 +pyzmq==25.1.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic - # notebook redis==4.2.2 # via feast (setup.py) -regex==2023.5.5 +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2023.8.8 # via feast (setup.py) requests==2.31.0 # via @@ -801,6 +807,7 @@ requests==2.31.0 # google-cloud-bigquery # google-cloud-storage # great-expectations + # jupyterlab-server # kubernetes # moto # msal @@ -815,7 +822,7 @@ requests-oauthlib==1.3.1 # google-auth-oauthlib # kubernetes # msrest -responses==0.23.1 +responses==0.23.3 # via moto rfc3339-validator==0.1.4 # via @@ -825,26 +832,26 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.0.0 +rockset==2.1.0 # via feast (setup.py) +rpds-py==0.10.2 + # via + # jsonschema + # referencing rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -s3transfer==0.6.1 +s3transfer==0.6.2 # via boto3 -scipy==1.10.1 +scipy==1.11.2 # via great-expectations send2trash==1.8.2 - # via - # jupyter-server - # nbclassic - # notebook + # via jupyter-server six==1.16.0 # via # asttokens # azure-core - # azure-identity # bleach # cassandra-driver # geomet @@ -866,29 +873,35 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.0.4 +snowflake-connector-python[pandas]==3.1.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python -soupsieve==2.4.1 +soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) -sphinxcontrib-applehelp==1.0.4 + # via + # feast (setup.py) + # sphinxcontrib-applehelp + # sphinxcontrib-devhelp + # sphinxcontrib-htmlhelp + # sphinxcontrib-qthelp + # sphinxcontrib-serializinghtml +sphinxcontrib-applehelp==1.0.7 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.5 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.4 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==1.0.6 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 # via sphinx -sqlalchemy[mypy]==1.4.48 +sqlalchemy[mypy]==1.4.49 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy stack-data==0.6.2 # via ipython @@ -896,14 +909,12 @@ starlette==0.27.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) -tenacity==8.2.2 +tenacity==8.2.3 # via feast (setup.py) terminado==0.17.1 # via # jupyter-server # jupyter-server-terminals - # nbclassic - # notebook testcontainers==3.7.1 # via feast (setup.py) thriftpy2==0.4.16 @@ -917,23 +928,27 @@ tomli==2.0.1 # black # build # coverage + # jupyterlab # mypy + # pip-tools # pyproject-hooks # pytest +tomlkit==0.12.1 + # via snowflake-connector-python toolz==0.12.0 # via # altair # dask # partd -tornado==6.3.2 +tornado==6.3.3 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # notebook # terminado -tqdm==4.65.0 +tqdm==4.66.1 # via # feast (setup.py) # great-expectations @@ -947,12 +962,11 @@ traitlets==5.9.0 # jupyter-core # jupyter-events # jupyter-server + # jupyterlab # matplotlib-inline - # nbclassic # nbclient # nbconvert # nbformat - # notebook trino==0.326.0 # via feast (setup.py) typeguard==2.13.3 @@ -961,42 +975,46 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19.7 +types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.2.0.0 +types-pyopenssl==23.2.0.2 # via types-redis -types-python-dateutil==2.8.19.13 +types-python-dateutil==2.8.19.14 # via feast (setup.py) -types-pytz==2023.3.0.0 +types-pytz==2023.3.0.1 # via feast (setup.py) -types-pyyaml==6.0.12.10 +types-pyyaml==6.0.12.11 # via # feast (setup.py) # responses -types-redis==4.5.5.2 +types-redis==4.6.0.5 # via feast (setup.py) -types-requests==2.31.0.1 +types-requests==2.31.0.2 # via feast (setup.py) -types-setuptools==67.8.0.0 +types-setuptools==68.2.0.0 # via feast (setup.py) -types-tabulate==0.9.0.2 +types-tabulate==0.9.0.3 # via feast (setup.py) -types-urllib3==1.26.25.13 +types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via + # async-lru # azure-core # azure-storage-blob + # fastapi + # filelock # great-expectations # mypy # pydantic # snowflake-connector-python # sqlalchemy2-stubs + # uvicorn tzlocal==5.0.1 # via # great-expectations # trino -uri-template==1.2.0 +uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client @@ -1013,15 +1031,17 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.22.0 +uvicorn[standard]==0.23.2 # via feast (setup.py) uvloop==0.17.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit volatile==2.1.0 # via bowler -watchfiles==0.19.0 +watchfiles==0.20.0 # via uvicorn wcwidth==0.2.6 # via prompt-toolkit @@ -1031,18 +1051,18 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.5.2 +websocket-client==1.6.2 # via # docker # jupyter-server # kubernetes websockets==11.0.3 # via uvicorn -werkzeug==2.3.4 +werkzeug==2.3.7 # via moto -wheel==0.40.0 +wheel==0.41.2 # via pip-tools -widgetsnbextension==4.0.7 +widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 # via @@ -1052,7 +1072,7 @@ xmltodict==0.13.0 # via moto yarl==1.9.2 # via aiohttp -zipp==3.15.0 +zipp==3.16.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index d9b70d51974..4140bea9d0f 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt # -anyio==3.7.0 +anyio==4.0.0 # via # httpcore # starlette @@ -15,16 +15,17 @@ attrs==23.1.0 # via # bowler # jsonschema + # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.5.7 +certifi==2023.7.22 # via # httpcore # httpx # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.7 # via # bowler # dask @@ -35,39 +36,43 @@ cloudpickle==2.2.1 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.5.1 +dask==2023.9.1 # via feast (setup.py) -dill==0.3.6 +dill==0.3.7 # via feast (setup.py) -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via anyio -fastapi==0.95.2 +fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.1 +fastavro==1.8.2 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.5.0 +fsspec==2023.9.0 # via dask -greenlet==2.0.2 - # via sqlalchemy -grpcio==1.54.2 +grpcio==1.57.0 # via # feast (setup.py) + # grpcio-health-checking # grpcio-reflection -grpcio-reflection==1.54.2 + # grpcio-tools +grpcio-health-checking==1.57.0 + # via feast (setup.py) +grpcio-reflection==1.57.0 # via feast (setup.py) -gunicorn==20.1.0 +grpcio-tools==1.57.0 + # via feast (setup.py) +gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.17.2 +httpcore==0.17.3 # via httpx -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 # via feast (setup.py) @@ -76,32 +81,38 @@ idna==3.4 # anyio # httpx # requests -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via dask jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.17.3 +jsonschema==4.19.0 # via feast (setup.py) +jsonschema-specifications==2023.7.1 + # via jsonschema locket==1.0.0 # via partd markupsafe==2.1.3 # via jinja2 -mmh3==4.0.0 +mmh3==4.0.1 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.3.0 +mypy==1.5.1 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -numpy==1.24.3 +mypy-protobuf==3.1 + # via feast (setup.py) +numpy==1.25.2 # via # feast (setup.py) # pandas # pandavro # pyarrow packaging==23.1 - # via dask + # via + # dask + # gunicorn pandas==1.5.3 # via # feast (setup.py) @@ -110,36 +121,45 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.4.0 # via dask -proto-plus==1.22.2 +proto-plus==1.22.3 # via feast (setup.py) -protobuf==4.23.2 +protobuf==4.23.3 # via # feast (setup.py) + # grpcio-health-checking # grpcio-reflection + # grpcio-tools + # mypy-protobuf # proto-plus pyarrow==11.0.0 # via feast (setup.py) -pydantic==1.10.8 +pydantic==1.10.12 # via # fastapi # feast (setup.py) -pygments==2.15.1 +pygments==2.16.1 # via feast (setup.py) -pyrsistent==0.19.3 - # via jsonschema python-dateutil==2.8.2 # via pandas python-dotenv==1.0.0 # via uvicorn -pytz==2023.3 +pytz==2023.3.post1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications requests==2.31.0 # via feast (setup.py) +rpds-py==0.10.2 + # via + # jsonschema + # referencing six==1.16.0 # via # pandavro @@ -149,15 +169,15 @@ sniffio==1.3.0 # anyio # httpcore # httpx -sqlalchemy[mypy]==1.4.48 +sqlalchemy[mypy]==1.4.49 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy starlette==0.27.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) -tenacity==8.2.2 +tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -167,26 +187,33 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.65.0 +tqdm==4.66.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -typing-extensions==4.6.3 +types-protobuf==4.24.0.1 + # via mypy-protobuf +typing-extensions==4.7.1 # via + # fastapi # mypy # pydantic # sqlalchemy2-stubs -urllib3==2.0.2 + # uvicorn +urllib3==2.0.4 # via requests -uvicorn[standard]==0.22.0 +uvicorn[standard]==0.23.2 # via feast (setup.py) uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.19.0 +watchfiles==0.20.0 # via uvicorn websockets==11.0.3 # via uvicorn -zipp==3.15.0 +zipp==3.16.2 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index de814374b68..9dfefc21081 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -8,7 +8,7 @@ adal==1.2.7 # via msrestazure adlfs==0.5.9 # via feast (setup.py) -aiohttp==3.8.4 +aiohttp==3.8.5 # via # adlfs # gcsfs @@ -18,7 +18,7 @@ alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations -anyio==3.7.0 +anyio==4.0.0 # via # httpcore # jupyter-server @@ -30,11 +30,8 @@ appnope==0.1.3 # via # ipykernel # ipython -argon2-cffi==21.3.0 - # via - # jupyter-server - # nbclassic - # notebook +argon2-cffi==23.1.0 + # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 @@ -45,9 +42,11 @@ asn1crypto==1.5.1 # snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.2.1 +asttokens==2.4.0 # via stack-data -async-timeout==4.0.2 +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 # via # aiohttp # redis @@ -56,9 +55,10 @@ attrs==23.1.0 # aiohttp # bowler # jsonschema + # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.27.0 +azure-core==1.29.3 # via # adlfs # azure-identity @@ -66,16 +66,18 @@ azure-core==1.27.0 # msrest azure-datalake-store==0.0.53 # via adlfs -azure-identity==1.13.0 +azure-identity==1.14.0 # via # adlfs # feast (setup.py) -azure-storage-blob==12.16.0 +azure-storage-blob==12.17.0 # via # adlfs # feast (setup.py) babel==2.12.1 - # via sphinx + # via + # jupyterlab-server + # sphinx backcall==0.2.0 # via ipython backports-zoneinfo==0.2.1 @@ -88,30 +90,30 @@ black==22.12.0 # via feast (setup.py) bleach==6.0.0 # via nbconvert -boto3==1.26.146 +boto3==1.28.42 # via # feast (setup.py) # moto -botocore==1.29.146 +botocore==1.31.42 # via # boto3 # moto # s3transfer bowler==0.9.0 # via feast (setup.py) -build==0.10.0 +build==1.0.3 # via # feast (setup.py) # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.0 +cachecontrol==0.13.1 # via firebase-admin cachetools==5.3.1 # via google-auth -cassandra-driver==3.27.0 +cassandra-driver==3.28.0 # via feast (setup.py) -certifi==2023.5.7 +certifi==2023.7.22 # via # httpcore # httpx @@ -126,14 +128,14 @@ cffi==1.15.1 # azure-datalake-store # cryptography # snowflake-connector-python -cfgv==3.3.1 +cfgv==3.4.0 # via pre-commit -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests # snowflake-connector-python -click==8.1.3 +click==8.1.7 # via # black # bowler @@ -150,16 +152,17 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.1.3 - # via ipykernel -coverage[toml]==7.2.7 +comm==0.1.4 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.3.1 # via pytest-cov -cryptography==40.0.2 +cryptography==41.0.3 # via # adal # azure-identity # azure-storage-blob - # cassandra-driver # feast (setup.py) # great-expectations # moto @@ -173,7 +176,7 @@ dask==2023.5.0 # via feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery -debugpy==1.6.7 +debugpy==1.6.7.post1 # via ipykernel decorator==5.1.1 # via @@ -185,12 +188,12 @@ deprecated==1.2.14 # via redis deprecation==2.1.0 # via testcontainers -dill==0.3.6 +dill==0.3.7 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.7 # via virtualenv docker==6.1.3 # via @@ -200,23 +203,23 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via # anyio # pytest -execnet==1.9.0 +execnet==2.0.2 # via pytest-xdist executing==1.2.0 # via stack-data -fastapi==0.95.2 +fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.1 +fastavro==1.8.2 # via # feast (setup.py) # pandavro -fastjsonschema==2.17.1 +fastjsonschema==2.18.0 # via nbformat -filelock==3.12.0 +filelock==3.12.3 # via # snowflake-connector-python # virtualenv @@ -228,7 +231,7 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal @@ -243,7 +246,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.0 +google-api-core[grpc]==2.11.1 # via # feast (setup.py) # firebase-admin @@ -255,9 +258,9 @@ google-api-core[grpc]==2.11.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.88.0 +google-api-python-client==2.98.0 # via firebase-admin -google-auth==2.19.1 +google-auth==2.22.0 # via # gcsfs # google-api-core @@ -271,35 +274,35 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-bigquery[pandas]==3.11.0 +google-cloud-bigquery[pandas]==3.11.4 # via feast (setup.py) -google-cloud-bigquery-storage==2.20.0 +google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) -google-cloud-bigtable==2.18.1 +google-cloud-bigtable==2.21.0 # via feast (setup.py) -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.15.2 +google-cloud-datastore==2.18.0 # via feast (setup.py) google-cloud-firestore==2.11.1 # via firebase-admin -google-cloud-storage==2.9.0 +google-cloud-storage==2.10.0 # via # feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.5.0 +google-resumable-media==2.6.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.59.0 +googleapis-common-protos[grpc]==1.60.0 # via # feast (setup.py) # google-api-core @@ -307,30 +310,31 @@ googleapis-common-protos[grpc]==1.59.0 # grpcio-status great-expectations==0.15.50 # via feast (setup.py) -greenlet==2.0.2 - # via sqlalchemy grpc-google-iam-v1==0.12.6 # via google-cloud-bigtable -grpcio==1.54.2 +grpcio==1.57.0 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos # grpc-google-iam-v1 + # grpcio-health-checking # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.54.2 +grpcio-health-checking==1.57.0 + # via feast (setup.py) +grpcio-reflection==1.57.0 # via feast (setup.py) -grpcio-status==1.54.2 +grpcio-status==1.57.0 # via google-api-core -grpcio-testing==1.54.2 +grpcio-testing==1.57.0 # via feast (setup.py) -grpcio-tools==1.54.2 +grpcio-tools==1.57.0 # via feast (setup.py) -gunicorn==20.1.0 +gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 # via @@ -338,21 +342,21 @@ h11==0.14.0 # uvicorn happybase==1.2.0 # via feast (setup.py) -hazelcast-python-client==5.2.0 +hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.2.3 # via feast (setup.py) -httpcore==0.17.2 +httpcore==0.17.3 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 # via feast (setup.py) -identify==2.5.24 +identify==2.5.27 # via pre-commit idna==3.4 # via @@ -364,32 +368,32 @@ idna==3.4 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via + # build # dask # great-expectations # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server # nbconvert # sphinx -importlib-resources==5.12.0 - # via jsonschema +importlib-resources==6.0.1 + # via + # jsonschema + # jsonschema-specifications + # jupyterlab iniconfig==2.0.0 # via pytest -ipykernel==6.23.1 - # via - # ipywidgets - # nbclassic - # notebook +ipykernel==6.25.2 + # via jupyterlab ipython==8.12.2 # via # great-expectations # ipykernel # ipywidgets -ipython-genutils==0.2.0 - # via - # nbclassic - # notebook -ipywidgets==8.0.6 +ipywidgets==8.1.0 # via great-expectations isodate==0.6.1 # via @@ -399,7 +403,7 @@ isoduration==20.11.0 # via jsonschema isort==5.12.0 # via feast (setup.py) -jedi==0.18.2 +jedi==0.19.0 # via ipython jinja2==3.1.2 # via @@ -407,56 +411,69 @@ jinja2==3.1.2 # feast (setup.py) # great-expectations # jupyter-server + # jupyterlab + # jupyterlab-server # moto - # nbclassic # nbconvert - # notebook # sphinx jmespath==1.0.1 # via # boto3 # botocore -jsonpatch==1.32 +json5==0.9.14 + # via jupyterlab-server +jsonpatch==1.33 # via great-expectations -jsonpointer==2.3 +jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.17.3 +jsonschema[format-nongpl]==4.19.0 # via # altair # feast (setup.py) # great-expectations # jupyter-events + # jupyterlab-server # nbformat -jupyter-client==8.2.0 +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==8.3.1 # via # ipykernel # jupyter-server - # nbclassic # nbclient - # notebook -jupyter-core==5.3.0 +jupyter-core==5.3.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # nbclient # nbconvert # nbformat - # notebook -jupyter-events==0.6.3 +jupyter-events==0.7.0 # via jupyter-server -jupyter-server==2.6.0 +jupyter-lsp==2.2.0 + # via jupyterlab +jupyter-server==2.7.3 # via - # nbclassic + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server +jupyterlab==4.0.5 + # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-widgets==3.0.7 +jupyterlab-server==2.24.0 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.8 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -469,7 +486,7 @@ markupsafe==2.1.3 # jinja2 # nbconvert # werkzeug -marshmallow==3.19.0 +marshmallow==3.20.1 # via great-expectations matplotlib-inline==0.1.6 # via @@ -479,19 +496,19 @@ mccabe==0.7.0 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.5 +mistune==3.0.1 # via # great-expectations # nbconvert -mmh3==4.0.0 +mmh3==4.0.1 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.1.10 +moto==4.2.2 # via feast (setup.py) -msal==1.22.0 +msal==1.23.0 # via # azure-datalake-store # azure-identity @@ -508,7 +525,7 @@ multidict==6.0.4 # via # aiohttp # yarl -multiprocess==0.70.14 +multiprocess==0.70.15 # via bytewax mypy==0.982 # via @@ -520,37 +537,29 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.1 # via feast (setup.py) -mysqlclient==2.1.1 +mysqlclient==2.2.0 # via feast (setup.py) -nbclassic==1.0.0 - # via notebook nbclient==0.8.0 # via nbconvert -nbconvert==7.4.0 - # via - # jupyter-server - # nbclassic - # notebook -nbformat==5.9.0 +nbconvert==7.8.0 + # via jupyter-server +nbformat==5.9.2 # via # great-expectations # jupyter-server - # nbclassic # nbclient # nbconvert - # notebook -nest-asyncio==1.5.6 - # via - # ipykernel - # nbclassic - # notebook +nest-asyncio==1.5.7 + # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==6.5.4 +notebook==7.0.3 # via great-expectations notebook-shim==0.2.3 - # via nbclassic -numpy==1.24.3 + # via + # jupyterlab + # notebook +numpy==1.24.4 # via # altair # db-dtypes @@ -564,7 +573,7 @@ oauthlib==3.2.2 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python -overrides==7.3.1 +overrides==7.4.0 # via jupyter-server packaging==23.1 # via @@ -575,8 +584,11 @@ packaging==23.1 # docker # google-cloud-bigquery # great-expectations + # gunicorn # ipykernel # jupyter-server + # jupyterlab + # jupyterlab-server # marshmallow # nbconvert # pytest @@ -600,7 +612,7 @@ parso==0.8.3 # via jedi partd==1.4.0 # via dask -pathspec==0.11.1 +pathspec==0.11.2 # via black pbr==5.11.1 # via mock @@ -608,31 +620,29 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.13.0 +pip-tools==7.3.0 # via feast (setup.py) pkgutil-resolve-name==1.3.10 # via jsonschema -platformdirs==3.5.1 +platformdirs==3.8.1 # via # black # jupyter-core + # snowflake-connector-python # virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via pytest ply==3.11 # via thriftpy2 portalocker==2.7.0 # via msal-extensions -pre-commit==3.3.2 +pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.17.0 - # via - # jupyter-server - # nbclassic - # notebook -prompt-toolkit==3.0.38 +prometheus-client==0.17.1 + # via jupyter-server +prompt-toolkit==3.0.39 # via ipython -proto-plus==1.22.2 +proto-plus==1.22.3 # via # feast (setup.py) # google-cloud-bigquery @@ -640,7 +650,7 @@ proto-plus==1.22.2 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.2 +protobuf==4.23.3 # via # feast (setup.py) # google-api-core @@ -651,6 +661,7 @@ protobuf==4.23.2 # google-cloud-firestore # googleapis-common-protos # grpc-google-iam-v1 + # grpcio-health-checking # grpcio-reflection # grpcio-status # grpcio-testing @@ -661,7 +672,7 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg2-binary==2.9.6 +psycopg2-binary==2.9.7 # via feast (setup.py) ptyprocess==0.7.0 # via @@ -695,43 +706,41 @@ pycparser==2.21 # via cffi pycryptodomex==3.18.0 # via snowflake-connector-python -pydantic==1.10.8 +pydantic==1.10.12 # via # fastapi # feast (setup.py) # great-expectations pyflakes==3.0.1 # via flake8 -pygments==2.15.1 +pygments==2.16.1 # via # feast (setup.py) # ipython # nbconvert # sphinx -pyjwt[crypto]==2.7.0 +pyjwt[crypto]==2.8.0 # via # adal # msal # snowflake-connector-python pymssql==2.2.8 # via feast (setup.py) -pymysql==1.0.3 +pymysql==1.1.0 # via feast (setup.py) pyodbc==4.0.39 # via feast (setup.py) pyopenssl==23.2.0 # via snowflake-connector-python -pyparsing==3.0.9 +pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 # via build -pyrsistent==0.19.3 - # via jsonschema -pyspark==3.4.0 +pyspark==3.4.1 # via feast (setup.py) -pytest==7.3.1 +pytest==7.4.1 # via # feast (setup.py) # pytest-benchmark @@ -772,7 +781,7 @@ python-dotenv==1.0.0 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3 +pytz==2023.3.post1 # via # babel # great-expectations @@ -788,16 +797,19 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.0 +pyzmq==25.1.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic - # notebook redis==4.2.2 # via feast (setup.py) -regex==2023.5.5 +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2023.8.8 # via feast (setup.py) requests==2.31.0 # via @@ -813,6 +825,7 @@ requests==2.31.0 # google-cloud-bigquery # google-cloud-storage # great-expectations + # jupyterlab-server # kubernetes # moto # msal @@ -827,7 +840,7 @@ requests-oauthlib==1.3.1 # google-auth-oauthlib # kubernetes # msrest -responses==0.23.1 +responses==0.23.3 # via moto rfc3339-validator==0.1.4 # via @@ -837,28 +850,28 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.0.0 +rockset==2.1.0 # via feast (setup.py) +rpds-py==0.10.2 + # via + # jsonschema + # referencing rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.7 # via ruamel-yaml -s3transfer==0.6.1 +s3transfer==0.6.2 # via boto3 scipy==1.10.1 # via great-expectations send2trash==1.8.2 - # via - # jupyter-server - # nbclassic - # notebook + # via jupyter-server six==1.16.0 # via # asttokens # azure-core - # azure-identity # bleach # cassandra-driver # geomet @@ -880,11 +893,11 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.0.4 +snowflake-connector-python[pandas]==3.1.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python -soupsieve==2.4.1 +soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 # via feast (setup.py) @@ -900,9 +913,9 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.48 +sqlalchemy[mypy]==1.4.49 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy stack-data==0.6.2 # via ipython @@ -910,14 +923,12 @@ starlette==0.27.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) -tenacity==8.2.2 +tenacity==8.2.3 # via feast (setup.py) terminado==0.17.1 # via # jupyter-server # jupyter-server-terminals - # nbclassic - # notebook testcontainers==3.7.1 # via feast (setup.py) thriftpy2==0.4.16 @@ -931,23 +942,27 @@ tomli==2.0.1 # black # build # coverage + # jupyterlab # mypy + # pip-tools # pyproject-hooks # pytest +tomlkit==0.12.1 + # via snowflake-connector-python toolz==0.12.0 # via # altair # dask # partd -tornado==6.3.2 +tornado==6.3.3 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # notebook # terminado -tqdm==4.65.0 +tqdm==4.66.1 # via # feast (setup.py) # great-expectations @@ -961,12 +976,11 @@ traitlets==5.9.0 # jupyter-core # jupyter-events # jupyter-server + # jupyterlab # matplotlib-inline - # nbclassic # nbclient # nbconvert # nbformat - # notebook trino==0.326.0 # via feast (setup.py) typeguard==2.13.3 @@ -975,33 +989,36 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19.7 +types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.2.0.0 +types-pyopenssl==23.2.0.2 # via types-redis -types-python-dateutil==2.8.19.13 +types-python-dateutil==2.8.19.14 # via feast (setup.py) -types-pytz==2023.3.0.0 +types-pytz==2023.3.0.1 # via feast (setup.py) -types-pyyaml==6.0.12.10 +types-pyyaml==6.0.12.11 # via # feast (setup.py) # responses -types-redis==4.5.5.2 +types-redis==4.6.0.5 # via feast (setup.py) -types-requests==2.31.0.1 +types-requests==2.31.0.2 # via feast (setup.py) -types-setuptools==67.8.0.0 +types-setuptools==68.2.0.0 # via feast (setup.py) -types-tabulate==0.9.0.2 +types-tabulate==0.9.0.3 # via feast (setup.py) -types-urllib3==1.26.25.13 +types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via + # async-lru # azure-core # azure-storage-blob # black + # fastapi + # filelock # great-expectations # ipython # mypy @@ -1009,11 +1026,12 @@ typing-extensions==4.6.3 # snowflake-connector-python # sqlalchemy2-stubs # starlette + # uvicorn tzlocal==5.0.1 # via # great-expectations # trino -uri-template==1.2.0 +uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client @@ -1030,15 +1048,17 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.22.0 +uvicorn[standard]==0.23.2 # via feast (setup.py) uvloop==0.17.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit volatile==2.1.0 # via bowler -watchfiles==0.19.0 +watchfiles==0.20.0 # via uvicorn wcwidth==0.2.6 # via prompt-toolkit @@ -1048,18 +1068,18 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.5.2 +websocket-client==1.6.2 # via # docker # jupyter-server # kubernetes websockets==11.0.3 # via uvicorn -werkzeug==2.3.4 +werkzeug==2.3.7 # via moto -wheel==0.40.0 +wheel==0.41.2 # via pip-tools -widgetsnbextension==4.0.7 +widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 # via @@ -1069,7 +1089,7 @@ xmltodict==0.13.0 # via moto yarl==1.9.2 # via aiohttp -zipp==3.15.0 +zipp==3.16.2 # via # importlib-metadata # importlib-resources diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 55ee6349048..636f886133d 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=sdk/python/requirements/py3.8-requirements.txt # -anyio==3.7.0 +anyio==4.0.0 # via # httpcore # starlette @@ -15,16 +15,17 @@ attrs==23.1.0 # via # bowler # jsonschema + # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.5.7 +certifi==2023.7.22 # via # httpcore # httpx # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.7 # via # bowler # dask @@ -37,37 +38,41 @@ colorama==0.4.6 # via feast (setup.py) dask==2023.5.0 # via feast (setup.py) -dill==0.3.6 +dill==0.3.7 # via feast (setup.py) -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via anyio -fastapi==0.95.2 +fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.1 +fastavro==1.8.2 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.5.0 +fsspec==2023.9.0 # via dask -greenlet==2.0.2 - # via sqlalchemy -grpcio==1.54.2 +grpcio==1.57.0 # via # feast (setup.py) + # grpcio-health-checking # grpcio-reflection -grpcio-reflection==1.54.2 + # grpcio-tools +grpcio-health-checking==1.57.0 + # via feast (setup.py) +grpcio-reflection==1.57.0 + # via feast (setup.py) +grpcio-tools==1.57.0 # via feast (setup.py) -gunicorn==20.1.0 +gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.17.2 +httpcore==0.17.3 # via httpx -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 # via feast (setup.py) @@ -76,34 +81,42 @@ idna==3.4 # anyio # httpx # requests -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via dask -importlib-resources==5.12.0 - # via jsonschema +importlib-resources==6.0.1 + # via + # jsonschema + # jsonschema-specifications jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.17.3 +jsonschema==4.19.0 # via feast (setup.py) +jsonschema-specifications==2023.7.1 + # via jsonschema locket==1.0.0 # via partd markupsafe==2.1.3 # via jinja2 -mmh3==4.0.0 +mmh3==4.0.1 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.3.0 +mypy==1.5.1 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -numpy==1.24.3 +mypy-protobuf==3.1 + # via feast (setup.py) +numpy==1.24.4 # via # feast (setup.py) # pandas # pandavro # pyarrow packaging==23.1 - # via dask + # via + # dask + # gunicorn pandas==1.5.3 # via # feast (setup.py) @@ -114,36 +127,45 @@ partd==1.4.0 # via dask pkgutil-resolve-name==1.3.10 # via jsonschema -proto-plus==1.22.2 +proto-plus==1.22.3 # via feast (setup.py) -protobuf==4.23.2 +protobuf==4.23.3 # via # feast (setup.py) + # grpcio-health-checking # grpcio-reflection + # grpcio-tools + # mypy-protobuf # proto-plus pyarrow==11.0.0 # via feast (setup.py) -pydantic==1.10.8 +pydantic==1.10.12 # via # fastapi # feast (setup.py) -pygments==2.15.1 +pygments==2.16.1 # via feast (setup.py) -pyrsistent==0.19.3 - # via jsonschema python-dateutil==2.8.2 # via pandas python-dotenv==1.0.0 # via uvicorn -pytz==2023.3 +pytz==2023.3.post1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications requests==2.31.0 # via feast (setup.py) +rpds-py==0.10.2 + # via + # jsonschema + # referencing six==1.16.0 # via # pandavro @@ -153,15 +175,15 @@ sniffio==1.3.0 # anyio # httpcore # httpx -sqlalchemy[mypy]==1.4.48 +sqlalchemy[mypy]==1.4.49 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy starlette==0.27.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) -tenacity==8.2.2 +tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -171,29 +193,36 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.65.0 +tqdm==4.66.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -typing-extensions==4.6.3 +types-protobuf==4.24.0.1 + # via mypy-protobuf +typing-extensions==4.7.1 # via + # fastapi # mypy # pydantic # sqlalchemy2-stubs # starlette -urllib3==2.0.2 + # uvicorn +urllib3==2.0.4 # via requests -uvicorn[standard]==0.22.0 +uvicorn[standard]==0.23.2 # via feast (setup.py) uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.19.0 +watchfiles==0.20.0 # via uvicorn websockets==11.0.3 # via uvicorn -zipp==3.15.0 +zipp==3.16.2 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index aa79b8ec3be..3992303d00e 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # @@ -8,7 +8,7 @@ adal==1.2.7 # via msrestazure adlfs==0.5.9 # via feast (setup.py) -aiohttp==3.8.4 +aiohttp==3.8.5 # via # adlfs # gcsfs @@ -18,7 +18,7 @@ alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations -anyio==3.7.0 +anyio==4.0.0 # via # httpcore # jupyter-server @@ -30,11 +30,8 @@ appnope==0.1.3 # via # ipykernel # ipython -argon2-cffi==21.3.0 - # via - # jupyter-server - # nbclassic - # notebook +argon2-cffi==23.1.0 + # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 @@ -45,9 +42,11 @@ asn1crypto==1.5.1 # snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.2.1 +asttokens==2.4.0 # via stack-data -async-timeout==4.0.2 +async-lru==2.0.4 + # via jupyterlab +async-timeout==4.0.3 # via # aiohttp # redis @@ -56,9 +55,10 @@ attrs==23.1.0 # aiohttp # bowler # jsonschema + # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.27.0 +azure-core==1.29.3 # via # adlfs # azure-identity @@ -66,16 +66,18 @@ azure-core==1.27.0 # msrest azure-datalake-store==0.0.53 # via adlfs -azure-identity==1.13.0 +azure-identity==1.14.0 # via # adlfs # feast (setup.py) -azure-storage-blob==12.16.0 +azure-storage-blob==12.17.0 # via # adlfs # feast (setup.py) babel==2.12.1 - # via sphinx + # via + # jupyterlab-server + # sphinx backcall==0.2.0 # via ipython beautifulsoup4==4.12.2 @@ -84,30 +86,30 @@ black==22.12.0 # via feast (setup.py) bleach==6.0.0 # via nbconvert -boto3==1.26.146 +boto3==1.28.42 # via # feast (setup.py) # moto -botocore==1.29.146 +botocore==1.31.42 # via # boto3 # moto # s3transfer bowler==0.9.0 # via feast (setup.py) -build==0.10.0 +build==1.0.3 # via # feast (setup.py) # pip-tools bytewax==0.15.1 # via feast (setup.py) -cachecontrol==0.13.0 +cachecontrol==0.13.1 # via firebase-admin cachetools==5.3.1 # via google-auth -cassandra-driver==3.27.0 +cassandra-driver==3.28.0 # via feast (setup.py) -certifi==2023.5.7 +certifi==2023.7.22 # via # httpcore # httpx @@ -122,14 +124,14 @@ cffi==1.15.1 # azure-datalake-store # cryptography # snowflake-connector-python -cfgv==3.3.1 +cfgv==3.4.0 # via pre-commit -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests # snowflake-connector-python -click==8.1.3 +click==8.1.7 # via # black # bowler @@ -146,16 +148,17 @@ colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.1.3 - # via ipykernel -coverage[toml]==7.2.7 +comm==0.1.4 + # via + # ipykernel + # ipywidgets +coverage[toml]==7.3.1 # via pytest-cov -cryptography==40.0.2 +cryptography==41.0.3 # via # adal # azure-identity # azure-storage-blob - # cassandra-driver # feast (setup.py) # great-expectations # moto @@ -165,11 +168,11 @@ cryptography==40.0.2 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.5.1 +dask==2023.9.1 # via feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery -debugpy==1.6.7 +debugpy==1.6.7.post1 # via ipykernel decorator==5.1.1 # via @@ -181,12 +184,12 @@ deprecated==1.2.14 # via redis deprecation==2.1.0 # via testcontainers -dill==0.3.6 +dill==0.3.7 # via # bytewax # feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.7 # via virtualenv docker==6.1.3 # via @@ -196,23 +199,24 @@ docutils==0.19 # via sphinx entrypoints==0.4 # via altair -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via # anyio + # ipython # pytest -execnet==1.9.0 +execnet==2.0.2 # via pytest-xdist executing==1.2.0 # via stack-data -fastapi==0.95.2 +fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.1 +fastavro==1.8.2 # via # feast (setup.py) # pandavro -fastjsonschema==2.17.1 +fastjsonschema==2.18.0 # via nbformat -filelock==3.12.0 +filelock==3.12.3 # via # snowflake-connector-python # virtualenv @@ -224,7 +228,7 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.3.3 +frozenlist==1.4.0 # via # aiohttp # aiosignal @@ -239,7 +243,7 @@ geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.0 +google-api-core[grpc]==2.11.1 # via # feast (setup.py) # firebase-admin @@ -251,9 +255,9 @@ google-api-core[grpc]==2.11.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.88.0 +google-api-python-client==2.98.0 # via firebase-admin -google-auth==2.19.1 +google-auth==2.22.0 # via # gcsfs # google-api-core @@ -267,35 +271,35 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-bigquery[pandas]==3.11.0 +google-cloud-bigquery[pandas]==3.11.4 # via feast (setup.py) -google-cloud-bigquery-storage==2.20.0 +google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) -google-cloud-bigtable==2.18.1 +google-cloud-bigtable==2.21.0 # via feast (setup.py) -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via # google-cloud-bigquery # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.15.2 +google-cloud-datastore==2.18.0 # via feast (setup.py) google-cloud-firestore==2.11.1 # via firebase-admin -google-cloud-storage==2.9.0 +google-cloud-storage==2.10.0 # via # feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.5.0 +google-resumable-media==2.6.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.59.0 +googleapis-common-protos[grpc]==1.60.0 # via # feast (setup.py) # google-api-core @@ -303,30 +307,31 @@ googleapis-common-protos[grpc]==1.59.0 # grpcio-status great-expectations==0.15.50 # via feast (setup.py) -greenlet==2.0.2 - # via sqlalchemy grpc-google-iam-v1==0.12.6 # via google-cloud-bigtable -grpcio==1.54.2 +grpcio==1.57.0 # via # feast (setup.py) # google-api-core # google-cloud-bigquery # googleapis-common-protos # grpc-google-iam-v1 + # grpcio-health-checking # grpcio-reflection # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.54.2 +grpcio-health-checking==1.57.0 # via feast (setup.py) -grpcio-status==1.54.2 +grpcio-reflection==1.57.0 + # via feast (setup.py) +grpcio-status==1.57.0 # via google-api-core -grpcio-testing==1.54.2 +grpcio-testing==1.57.0 # via feast (setup.py) -grpcio-tools==1.54.2 +grpcio-tools==1.57.0 # via feast (setup.py) -gunicorn==20.1.0 +gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 # via @@ -334,21 +339,21 @@ h11==0.14.0 # uvicorn happybase==1.2.0 # via feast (setup.py) -hazelcast-python-client==5.2.0 +hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.2.3 # via feast (setup.py) -httpcore==0.17.2 +httpcore==0.17.3 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 # via feast (setup.py) -identify==2.5.24 +identify==2.5.27 # via pre-commit idna==3.4 # via @@ -360,30 +365,27 @@ idna==3.4 # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via + # build # dask # great-expectations # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server # nbconvert # sphinx iniconfig==2.0.0 # via pytest -ipykernel==6.23.1 - # via - # ipywidgets - # nbclassic - # notebook -ipython==8.14.0 +ipykernel==6.25.2 + # via jupyterlab +ipython==8.15.0 # via # great-expectations # ipykernel # ipywidgets -ipython-genutils==0.2.0 - # via - # nbclassic - # notebook -ipywidgets==8.0.6 +ipywidgets==8.1.0 # via great-expectations isodate==0.6.1 # via @@ -393,7 +395,7 @@ isoduration==20.11.0 # via jsonschema isort==5.12.0 # via feast (setup.py) -jedi==0.18.2 +jedi==0.19.0 # via ipython jinja2==3.1.2 # via @@ -401,56 +403,69 @@ jinja2==3.1.2 # feast (setup.py) # great-expectations # jupyter-server + # jupyterlab + # jupyterlab-server # moto - # nbclassic # nbconvert - # notebook # sphinx jmespath==1.0.1 # via # boto3 # botocore -jsonpatch==1.32 +json5==0.9.14 + # via jupyterlab-server +jsonpatch==1.33 # via great-expectations -jsonpointer==2.3 +jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.17.3 +jsonschema[format-nongpl]==4.19.0 # via # altair # feast (setup.py) # great-expectations # jupyter-events + # jupyterlab-server # nbformat -jupyter-client==8.2.0 +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==8.3.1 # via # ipykernel # jupyter-server - # nbclassic # nbclient - # notebook -jupyter-core==5.3.0 +jupyter-core==5.3.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # nbclient # nbconvert # nbformat - # notebook -jupyter-events==0.6.3 +jupyter-events==0.7.0 # via jupyter-server -jupyter-server==2.6.0 +jupyter-lsp==2.2.0 + # via jupyterlab +jupyter-server==2.7.3 # via - # nbclassic + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server +jupyterlab==4.0.5 + # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-widgets==3.0.7 +jupyterlab-server==2.24.0 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.8 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -463,7 +478,7 @@ markupsafe==2.1.3 # jinja2 # nbconvert # werkzeug -marshmallow==3.19.0 +marshmallow==3.20.1 # via great-expectations matplotlib-inline==0.1.6 # via @@ -473,19 +488,19 @@ mccabe==0.7.0 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.5 +mistune==3.0.1 # via # great-expectations # nbconvert -mmh3==4.0.0 +mmh3==4.0.1 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.1.10 +moto==4.2.2 # via feast (setup.py) -msal==1.22.0 +msal==1.23.0 # via # azure-datalake-store # azure-identity @@ -502,7 +517,7 @@ multidict==6.0.4 # via # aiohttp # yarl -multiprocess==0.70.14 +multiprocess==0.70.15 # via bytewax mypy==0.982 # via @@ -514,37 +529,29 @@ mypy-extensions==1.0.0 # mypy mypy-protobuf==3.1 # via feast (setup.py) -mysqlclient==2.1.1 +mysqlclient==2.2.0 # via feast (setup.py) -nbclassic==1.0.0 - # via notebook nbclient==0.8.0 # via nbconvert -nbconvert==7.4.0 - # via - # jupyter-server - # nbclassic - # notebook -nbformat==5.9.0 +nbconvert==7.8.0 + # via jupyter-server +nbformat==5.9.2 # via # great-expectations # jupyter-server - # nbclassic # nbclient # nbconvert - # notebook -nest-asyncio==1.5.6 - # via - # ipykernel - # nbclassic - # notebook +nest-asyncio==1.5.7 + # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==6.5.4 +notebook==7.0.3 # via great-expectations notebook-shim==0.2.3 - # via nbclassic -numpy==1.24.3 + # via + # jupyterlab + # notebook +numpy==1.25.2 # via # altair # db-dtypes @@ -558,7 +565,7 @@ oauthlib==3.2.2 # via requests-oauthlib oscrypto==1.3.0 # via snowflake-connector-python -overrides==7.3.1 +overrides==7.4.0 # via jupyter-server packaging==23.1 # via @@ -569,8 +576,11 @@ packaging==23.1 # docker # google-cloud-bigquery # great-expectations + # gunicorn # ipykernel # jupyter-server + # jupyterlab + # jupyterlab-server # marshmallow # nbconvert # pytest @@ -594,7 +604,7 @@ parso==0.8.3 # via jedi partd==1.4.0 # via dask -pathspec==0.11.1 +pathspec==0.11.2 # via black pbr==5.11.1 # via mock @@ -602,29 +612,27 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.13.0 +pip-tools==7.3.0 # via feast (setup.py) -platformdirs==3.5.1 +platformdirs==3.8.1 # via # black # jupyter-core + # snowflake-connector-python # virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via pytest ply==3.11 # via thriftpy2 portalocker==2.7.0 # via msal-extensions -pre-commit==3.3.2 +pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.17.0 - # via - # jupyter-server - # nbclassic - # notebook -prompt-toolkit==3.0.38 +prometheus-client==0.17.1 + # via jupyter-server +prompt-toolkit==3.0.39 # via ipython -proto-plus==1.22.2 +proto-plus==1.22.3 # via # feast (setup.py) # google-cloud-bigquery @@ -632,7 +640,7 @@ proto-plus==1.22.2 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.2 +protobuf==4.23.3 # via # feast (setup.py) # google-api-core @@ -643,6 +651,7 @@ protobuf==4.23.2 # google-cloud-firestore # googleapis-common-protos # grpc-google-iam-v1 + # grpcio-health-checking # grpcio-reflection # grpcio-status # grpcio-testing @@ -653,7 +662,7 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg2-binary==2.9.6 +psycopg2-binary==2.9.7 # via feast (setup.py) ptyprocess==0.7.0 # via @@ -687,43 +696,41 @@ pycparser==2.21 # via cffi pycryptodomex==3.18.0 # via snowflake-connector-python -pydantic==1.10.8 +pydantic==1.10.12 # via # fastapi # feast (setup.py) # great-expectations pyflakes==3.0.1 # via flake8 -pygments==2.15.1 +pygments==2.16.1 # via # feast (setup.py) # ipython # nbconvert # sphinx -pyjwt[crypto]==2.7.0 +pyjwt[crypto]==2.8.0 # via # adal # msal # snowflake-connector-python pymssql==2.2.8 # via feast (setup.py) -pymysql==1.0.3 +pymysql==1.1.0 # via feast (setup.py) pyodbc==4.0.39 # via feast (setup.py) pyopenssl==23.2.0 # via snowflake-connector-python -pyparsing==3.0.9 +pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 # via build -pyrsistent==0.19.3 - # via jsonschema -pyspark==3.4.0 +pyspark==3.4.1 # via feast (setup.py) -pytest==7.3.1 +pytest==7.4.1 # via # feast (setup.py) # pytest-benchmark @@ -764,7 +771,7 @@ python-dotenv==1.0.0 # via uvicorn python-json-logger==2.0.7 # via jupyter-events -pytz==2023.3 +pytz==2023.3.post1 # via # great-expectations # pandas @@ -779,16 +786,19 @@ pyyaml==6.0.1 # pre-commit # responses # uvicorn -pyzmq==25.1.0 +pyzmq==25.1.1 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic - # notebook redis==4.2.2 # via feast (setup.py) -regex==2023.5.5 +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2023.8.8 # via feast (setup.py) requests==2.31.0 # via @@ -804,6 +814,7 @@ requests==2.31.0 # google-cloud-bigquery # google-cloud-storage # great-expectations + # jupyterlab-server # kubernetes # moto # msal @@ -818,7 +829,7 @@ requests-oauthlib==1.3.1 # google-auth-oauthlib # kubernetes # msrest -responses==0.23.1 +responses==0.23.3 # via moto rfc3339-validator==0.1.4 # via @@ -828,28 +839,28 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.0.0 +rockset==2.1.0 # via feast (setup.py) +rpds-py==0.10.2 + # via + # jsonschema + # referencing rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.7 # via ruamel-yaml -s3transfer==0.6.1 +s3transfer==0.6.2 # via boto3 -scipy==1.10.1 +scipy==1.11.2 # via great-expectations send2trash==1.8.2 - # via - # jupyter-server - # nbclassic - # notebook + # via jupyter-server six==1.16.0 # via # asttokens # azure-core - # azure-identity # bleach # cassandra-driver # geomet @@ -871,29 +882,35 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.0.4 +snowflake-connector-python[pandas]==3.1.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python -soupsieve==2.4.1 +soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 - # via feast (setup.py) -sphinxcontrib-applehelp==1.0.4 + # via + # feast (setup.py) + # sphinxcontrib-applehelp + # sphinxcontrib-devhelp + # sphinxcontrib-htmlhelp + # sphinxcontrib-qthelp + # sphinxcontrib-serializinghtml +sphinxcontrib-applehelp==1.0.7 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.5 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.4 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==1.0.6 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 # via sphinx -sqlalchemy[mypy]==1.4.48 +sqlalchemy[mypy]==1.4.49 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy stack-data==0.6.2 # via ipython @@ -901,14 +918,12 @@ starlette==0.27.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) -tenacity==8.2.2 +tenacity==8.2.3 # via feast (setup.py) terminado==0.17.1 # via # jupyter-server # jupyter-server-terminals - # nbclassic - # notebook testcontainers==3.7.1 # via feast (setup.py) thriftpy2==0.4.16 @@ -922,23 +937,27 @@ tomli==2.0.1 # black # build # coverage + # jupyterlab # mypy + # pip-tools # pyproject-hooks # pytest +tomlkit==0.12.1 + # via snowflake-connector-python toolz==0.12.0 # via # altair # dask # partd -tornado==6.3.2 +tornado==6.3.3 # via # ipykernel # jupyter-client # jupyter-server - # nbclassic + # jupyterlab # notebook # terminado -tqdm==4.65.0 +tqdm==4.66.1 # via # feast (setup.py) # great-expectations @@ -952,12 +971,11 @@ traitlets==5.9.0 # jupyter-core # jupyter-events # jupyter-server + # jupyterlab # matplotlib-inline - # nbclassic # nbclient # nbconvert # nbformat - # notebook trino==0.326.0 # via feast (setup.py) typeguard==2.13.3 @@ -966,33 +984,36 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19.7 +types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.2.0.0 +types-pyopenssl==23.2.0.2 # via types-redis -types-python-dateutil==2.8.19.13 +types-python-dateutil==2.8.19.14 # via feast (setup.py) -types-pytz==2023.3.0.0 +types-pytz==2023.3.0.1 # via feast (setup.py) -types-pyyaml==6.0.12.10 +types-pyyaml==6.0.12.11 # via # feast (setup.py) # responses -types-redis==4.5.5.2 +types-redis==4.6.0.5 # via feast (setup.py) -types-requests==2.31.0.1 +types-requests==2.31.0.2 # via feast (setup.py) -types-setuptools==67.8.0.0 +types-setuptools==68.2.0.0 # via feast (setup.py) -types-tabulate==0.9.0.2 +types-tabulate==0.9.0.3 # via feast (setup.py) -types-urllib3==1.26.25.13 +types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via + # async-lru # azure-core # azure-storage-blob # black + # fastapi + # filelock # great-expectations # ipython # mypy @@ -1000,11 +1021,12 @@ typing-extensions==4.6.3 # snowflake-connector-python # sqlalchemy2-stubs # starlette + # uvicorn tzlocal==5.0.1 # via # great-expectations # trino -uri-template==1.2.0 +uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client @@ -1021,15 +1043,17 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.22.0 +uvicorn[standard]==0.23.2 # via feast (setup.py) uvloop==0.17.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit volatile==2.1.0 # via bowler -watchfiles==0.19.0 +watchfiles==0.20.0 # via uvicorn wcwidth==0.2.6 # via prompt-toolkit @@ -1039,18 +1063,18 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.5.2 +websocket-client==1.6.2 # via # docker # jupyter-server # kubernetes websockets==11.0.3 # via uvicorn -werkzeug==2.3.4 +werkzeug==2.3.7 # via moto -wheel==0.40.0 +wheel==0.41.2 # via pip-tools -widgetsnbextension==4.0.7 +widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 # via @@ -1060,7 +1084,7 @@ xmltodict==0.13.0 # via moto yarl==1.9.2 # via aiohttp -zipp==3.15.0 +zipp==3.16.2 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 53b0ff0e584..7d30fd3452b 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt # -anyio==3.7.0 +anyio==4.0.0 # via # httpcore # starlette @@ -15,16 +15,17 @@ attrs==23.1.0 # via # bowler # jsonschema + # referencing bowler==0.9.0 # via feast (setup.py) -certifi==2023.5.7 +certifi==2023.7.22 # via # httpcore # httpx # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.7 # via # bowler # dask @@ -35,39 +36,43 @@ cloudpickle==2.2.1 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.5.1 +dask==2023.9.1 # via feast (setup.py) -dill==0.3.6 +dill==0.3.7 # via feast (setup.py) -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via anyio -fastapi==0.95.2 +fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.1 +fastavro==1.8.2 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.5.0 +fsspec==2023.9.0 # via dask -greenlet==2.0.2 - # via sqlalchemy -grpcio==1.54.2 +grpcio==1.57.0 # via # feast (setup.py) + # grpcio-health-checking # grpcio-reflection -grpcio-reflection==1.54.2 + # grpcio-tools +grpcio-health-checking==1.57.0 + # via feast (setup.py) +grpcio-reflection==1.57.0 # via feast (setup.py) -gunicorn==20.1.0 +grpcio-tools==1.57.0 + # via feast (setup.py) +gunicorn==21.2.0 # via feast (setup.py) h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.17.2 +httpcore==0.17.3 # via httpx -httptools==0.5.0 +httptools==0.6.0 # via uvicorn httpx==0.24.1 # via feast (setup.py) @@ -76,32 +81,38 @@ idna==3.4 # anyio # httpx # requests -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via dask jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.17.3 +jsonschema==4.19.0 # via feast (setup.py) +jsonschema-specifications==2023.7.1 + # via jsonschema locket==1.0.0 # via partd markupsafe==2.1.3 # via jinja2 -mmh3==4.0.0 +mmh3==4.0.1 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.3.0 +mypy==1.5.1 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -numpy==1.24.3 +mypy-protobuf==3.1.0 + # via feast (setup.py) +numpy==1.25.2 # via # feast (setup.py) # pandas # pandavro # pyarrow packaging==23.1 - # via dask + # via + # dask + # gunicorn pandas==1.5.3 # via # feast (setup.py) @@ -110,36 +121,45 @@ pandavro==1.5.2 # via feast (setup.py) partd==1.4.0 # via dask -proto-plus==1.22.2 +proto-plus==1.22.3 # via feast (setup.py) -protobuf==4.23.2 +protobuf==4.23.3 # via # feast (setup.py) + # grpcio-health-checking # grpcio-reflection + # grpcio-tools + # mypy-protobuf # proto-plus pyarrow==11.0.0 # via feast (setup.py) -pydantic==1.10.8 +pydantic==1.10.12 # via # fastapi # feast (setup.py) -pygments==2.15.1 +pygments==2.16.1 # via feast (setup.py) -pyrsistent==0.19.3 - # via jsonschema python-dateutil==2.8.2 # via pandas python-dotenv==1.0.0 # via uvicorn -pytz==2023.3 +pytz==2023.3.post1 # via pandas pyyaml==6.0.1 # via # dask # feast (setup.py) # uvicorn +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications requests==2.31.0 # via feast (setup.py) +rpds-py==0.10.2 + # via + # jsonschema + # referencing six==1.16.0 # via # pandavro @@ -149,15 +169,15 @@ sniffio==1.3.0 # anyio # httpcore # httpx -sqlalchemy[mypy]==1.4.48 +sqlalchemy[mypy]==1.4.49 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a34 +sqlalchemy2-stubs==0.0.2a35 # via sqlalchemy starlette==0.27.0 # via fastapi tabulate==0.9.0 # via feast (setup.py) -tenacity==8.2.2 +tenacity==8.2.3 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) @@ -167,27 +187,34 @@ toolz==0.12.0 # via # dask # partd -tqdm==4.65.0 +tqdm==4.66.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -typing-extensions==4.6.3 +types-protobuf==4.24.0.1 + # via mypy-protobuf +typing-extensions==4.7.1 # via + # fastapi # mypy # pydantic # sqlalchemy2-stubs # starlette -urllib3==2.0.2 + # uvicorn +urllib3==2.0.4 # via requests -uvicorn[standard]==0.22.0 +uvicorn[standard]==0.23.2 # via feast (setup.py) uvloop==0.17.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.19.0 +watchfiles==0.20.0 # via uvicorn websockets==11.0.3 # via uvicorn -zipp==3.15.0 +zipp==3.16.2 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.py b/setup.py index f10322ed8a3..699d394940e 100644 --- a/setup.py +++ b/setup.py @@ -46,8 +46,11 @@ "colorama>=0.3.9,<1", "dill~=0.3.0", "fastavro>=1.1.0,<2", - "grpcio>=1.47.0,<2", - "grpcio-reflection>=1.47.0,<2", + "grpcio>=1.56.2,<2", + "grpcio-tools>=1.56.2,<2", + "grpcio-reflection>=1.56.2,<2", + "grpcio-health-checking>=1.56.2,<2", + "mypy-protobuf==3.1", "Jinja2>=2,<4", "jsonschema", "mmh3", @@ -69,7 +72,7 @@ "toml>=0.10.0,<1", "tqdm>=4,<5", "typeguard==2.13.3", - "fastapi>=0.68.0,<1", + "fastapi>=0.68.0,<0.100", "uvicorn[standard]>=0.14.0,<1", "gunicorn", "dask>=2021.1.0", @@ -142,17 +145,16 @@ CI_REQUIRED = ( [ "build", + "virtualenv==20.23.0", "cryptography>=35.0,<42", - "flake8", + "flake8>=6.0.0,<6.1.0", "black>=22.6.0,<23", "isort>=5,<6", - "grpcio-tools>=1.47.0", - "grpcio-testing>=1.47.0", + "grpcio-testing>=1.56.2,<2", "minio==7.1.0", "mock==2.0.0", "moto", "mypy>=0.981,<0.990", - "mypy-protobuf==3.1", "avro==1.10.0", "gcsfs>=0.4.0,<=2022.01.0", "urllib3>=1.25.4,<2", @@ -170,7 +172,7 @@ "testcontainers>=3.5,<4", "adlfs==0.5.9", "firebase-admin>=5.2.0,<6", - "pre-commit", + "pre-commit<3.3.2", "assertpy==1.1", "pip-tools", "pybindgen", @@ -182,6 +184,7 @@ "types-requests", "types-setuptools", "types-tabulate", + "virtualenv<20.24.2" ] + GCP_REQUIRED + REDIS_REQUIRED @@ -380,8 +383,8 @@ def run(self): use_scm_version=use_scm_version, setup_requires=[ "setuptools_scm", - "grpcio>=1.47.0", - "grpcio-tools>=1.47.0", + "grpcio>=1.56.2,<2", + "grpcio-tools>=1.56.2,<2", "mypy-protobuf==3.1", "pybindgen==0.22.0", ], From 6474b4b0169dc9b3df8e8daecded2b1fad5ead58 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 7 Sep 2023 11:51:00 +0700 Subject: [PATCH 14/18] fix: Add NUMERIC to bq_to_feast type map (#3719) Signed-off-by: Hai Nguyen --- sdk/python/feast/type_map.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index d246fb40545..3f490690669 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -528,6 +528,7 @@ def bq_to_feast_value_type(bq_type_as_str: str) -> ValueType: "DATETIME": ValueType.UNIX_TIMESTAMP, "TIMESTAMP": ValueType.UNIX_TIMESTAMP, "INTEGER": ValueType.INT64, + "NUMERIC": ValueType.INT64, "INT64": ValueType.INT64, "STRING": ValueType.STRING, "FLOAT": ValueType.DOUBLE, From 0049356b3243a5c16760541520640343023f6ea7 Mon Sep 17 00:00:00 2001 From: Breno Costa <35263725+breno-costa@users.noreply.github.com> Date: Thu, 7 Sep 2023 07:31:28 +0200 Subject: [PATCH 15/18] chore: Widen redis requirement to make it easier to install alongside other requirements (#3755) * chore: wide redis dependency to make it easier to install alongside other requirements Signed-off-by: Breno Costa * update requirements Signed-off-by: Danny C --------- Signed-off-by: Breno Costa Signed-off-by: Danny C Co-authored-by: Danny C --- sdk/python/requirements/py3.10-ci-requirements.txt | 9 ++------- sdk/python/requirements/py3.8-ci-requirements.txt | 9 ++------- sdk/python/requirements/py3.9-ci-requirements.txt | 11 +++-------- setup.py | 2 +- 4 files changed, 8 insertions(+), 23 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index cb72fdaa350..a59553b4ac8 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -180,8 +180,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.14 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.7 @@ -577,7 +575,6 @@ packaging==23.1 # marshmallow # nbconvert # pytest - # redis # snowflake-connector-python # sphinx pandas==1.5.3 @@ -784,7 +781,7 @@ pyzmq==25.1.1 # ipykernel # jupyter-client # jupyter-server -redis==4.2.2 +redis==4.6.0 # via feast (setup.py) referencing==0.30.2 # via @@ -1065,9 +1062,7 @@ wheel==0.41.2 widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.13.0 # via moto yarl==1.9.2 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 9dfefc21081..b24172e890f 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -184,8 +184,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.14 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.7 @@ -592,7 +590,6 @@ packaging==23.1 # marshmallow # nbconvert # pytest - # redis # snowflake-connector-python # sphinx pandas==1.5.3 @@ -802,7 +799,7 @@ pyzmq==25.1.1 # ipykernel # jupyter-client # jupyter-server -redis==4.2.2 +redis==4.6.0 # via feast (setup.py) referencing==0.30.2 # via @@ -1082,9 +1079,7 @@ wheel==0.41.2 widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.13.0 # via moto yarl==1.9.2 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 3992303d00e..ad19f9e8bde 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -180,8 +180,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.14 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.7 @@ -527,7 +525,7 @@ mypy-extensions==1.0.0 # via # black # mypy -mypy-protobuf==3.1 +mypy-protobuf==3.1.0 # via feast (setup.py) mysqlclient==2.2.0 # via feast (setup.py) @@ -584,7 +582,6 @@ packaging==23.1 # marshmallow # nbconvert # pytest - # redis # snowflake-connector-python # sphinx pandas==1.5.3 @@ -791,7 +788,7 @@ pyzmq==25.1.1 # ipykernel # jupyter-client # jupyter-server -redis==4.2.2 +redis==4.6.0 # via feast (setup.py) referencing==0.30.2 # via @@ -1077,9 +1074,7 @@ wheel==0.41.2 widgetsnbextension==4.0.8 # via ipywidgets wrapt==1.15.0 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.13.0 # via moto yarl==1.9.2 diff --git a/setup.py b/setup.py index 699d394940e..573ab54d512 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ ] REDIS_REQUIRED = [ - "redis==4.2.2", + "redis>=4.2.2,<5", "hiredis>=2.0.0,<3", ] From f28ccc2b8f42bcca943d498ad583337d4cd70383 Mon Sep 17 00:00:00 2001 From: Malcolm Keyes <78397791+malcolmk181@users.noreply.github.com> Date: Thu, 7 Sep 2023 01:51:00 -0400 Subject: [PATCH 16/18] fix: Saved datasets no longer break CLI registry-dump command (#3717) fix: Access saved dataset proto correctly in list_saved_datasets Signed-off-by: Malcolm Keyes --- sdk/python/feast/infra/registry/proto_registry_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py index 2a275703dbb..c7eeea0f822 100644 --- a/sdk/python/feast/infra/registry/proto_registry_utils.py +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -214,7 +214,7 @@ def list_saved_datasets( ) -> List[SavedDataset]: saved_datasets = [] for saved_dataset in registry_proto.saved_datasets: - if saved_dataset.project == project: + if saved_dataset.spec.project == project: saved_datasets.append(SavedDataset.from_proto(saved_dataset)) return saved_datasets From ed7535e23d490249ca7d224fb88e53b98d496ec0 Mon Sep 17 00:00:00 2001 From: Dani Date: Thu, 7 Sep 2023 08:18:12 +0200 Subject: [PATCH 17/18] feat: Enhance customization of Trino connections when using Trino-based Offline Stores (#3699) * feat: Enhance customization of Trino connections when using Trino-based Offline Stores Signed-off-by: boliri * docs: Add new connection parameters to Trino Offline Store's reference Signed-off-by: boliri --------- Signed-off-by: boliri --- docs/reference/offline-stores/trino.md | 41 +++++ .../trino_offline_store/tests/data_source.py | 5 + .../contrib/trino_offline_store/trino.py | 164 ++++++++++++++---- .../trino_offline_store/trino_queries.py | 49 +++--- .../trino_offline_store/trino_source.py | 10 ++ 5 files changed, 209 insertions(+), 60 deletions(-) diff --git a/docs/reference/offline-stores/trino.md b/docs/reference/offline-stores/trino.md index 446db620e32..fd437a7aa67 100644 --- a/docs/reference/offline-stores/trino.md +++ b/docs/reference/offline-stores/trino.md @@ -27,6 +27,47 @@ offline_store: catalog: memory connector: type: memory + user: trino + source: feast-trino-offline-store + http-scheme: https + ssl-verify: false + x-trino-extra-credential-header: foo=bar, baz=qux + + # enables authentication in Trino connections, pick the one you need + # if you don't need authentication, you can safely remove the whole auth block + auth: + # Basic Auth + type: basic + config: + username: foo + password: $FOO + + # Certificate + type: certificate + config: + cert-file: /path/to/cert/file + key-file: /path/to/key/file + + # JWT + type: jwt + config: + token: $JWT_TOKEN + + # OAuth2 (no config required) + type: oauth2 + + # Kerberos + type: kerberos + config: + config-file: /path/to/kerberos/config/file + service-name: foo + mutual-authentication: true + force-preemptive: true + hostname-override: custom-hostname + sanitize-mutual-error-response: true + principal: principal-name + delegate: true + ca_bundle: /path/to/ca/bundle/file online_store: path: data/online_store.db ``` diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py index 67efa6a27f8..a5aa53df7ab 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/tests/data_source.py @@ -67,6 +67,11 @@ def __init__( catalog="memory", host="localhost", port=self.exposed_port, + source="trino-python-client", + http_scheme="http", + verify=False, + extra_credential=None, + auth=None, ) def teardown(self): diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index e0f73404ebe..f662cda9130 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -1,12 +1,18 @@ import uuid from datetime import date, datetime -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Literal, Optional, Tuple, Union import numpy as np import pandas as pd import pyarrow -from pydantic import StrictStr -from trino.auth import Authentication +from pydantic import Field, FilePath, SecretStr, StrictBool, StrictStr, root_validator +from trino.auth import ( + BasicAuthentication, + CertificateAuthentication, + JWTAuthentication, + KerberosAuthentication, + OAuth2Authentication, +) from feast.data_source import DataSource from feast.errors import InvalidEntityType @@ -32,6 +38,87 @@ from feast.usage import log_exceptions_and_usage +class BasicAuthModel(FeastConfigBaseModel): + username: StrictStr + password: SecretStr + + +class KerberosAuthModel(FeastConfigBaseModel): + config: Optional[FilePath] = Field(default=None, alias="config-file") + service_name: Optional[StrictStr] = Field(default=None, alias="service-name") + mutual_authentication: StrictBool = Field( + default=False, alias="mutual-authentication" + ) + force_preemptive: StrictBool = Field(default=False, alias="force-preemptive") + hostname_override: Optional[StrictStr] = Field( + default=None, alias="hostname-override" + ) + sanitize_mutual_error_response: StrictBool = Field( + default=True, alias="sanitize-mutual-error-response" + ) + principal: Optional[StrictStr] + delegate: StrictBool = False + ca_bundle: Optional[FilePath] = Field(default=None, alias="ca-bundle-file") + + +class JWTAuthModel(FeastConfigBaseModel): + token: SecretStr + + +class CertificateAuthModel(FeastConfigBaseModel): + cert: FilePath = Field(default=None, alias="cert-file") + key: FilePath = Field(default=None, alias="key-file") + + +CLASSES_BY_AUTH_TYPE = { + "kerberos": { + "auth_model": KerberosAuthModel, + "trino_auth": KerberosAuthentication, + }, + "basic": { + "auth_model": BasicAuthModel, + "trino_auth": BasicAuthentication, + }, + "jwt": { + "auth_model": JWTAuthModel, + "trino_auth": JWTAuthentication, + }, + "oauth2": { + "auth_model": None, + "trino_auth": OAuth2Authentication, + }, + "certificate": { + "auth_model": CertificateAuthModel, + "trino_auth": CertificateAuthentication, + }, +} + + +class AuthConfig(FeastConfigBaseModel): + type: Literal["kerberos", "basic", "jwt", "oauth2", "certificate"] + config: Optional[Dict[StrictStr, Any]] + + @root_validator + def config_only_nullable_for_oauth2(cls, values): + auth_type = values["type"] + auth_config = values["config"] + if auth_type != "oauth2" and auth_config is None: + raise ValueError(f"config cannot be null for auth type '{auth_type}'") + + return values + + def to_trino_auth(self): + auth_type = self.type + trino_auth_cls = CLASSES_BY_AUTH_TYPE[auth_type]["trino_auth"] + + if auth_type == "oauth2": + return trino_auth_cls() + + model_cls = CLASSES_BY_AUTH_TYPE[auth_type]["auth_model"] + model = model_cls(**self.config) + return trino_auth_cls(**model.dict()) + + class TrinoOfflineStoreConfig(FeastConfigBaseModel): """Online store config for Trino""" @@ -47,6 +134,23 @@ class TrinoOfflineStoreConfig(FeastConfigBaseModel): catalog: StrictStr """ Catalog of the Trino cluster """ + user: StrictStr + """ User of the Trino cluster """ + + source: Optional[StrictStr] = "trino-python-client" + """ ID of the feast's Trino Python client, useful for debugging """ + + http_scheme: Literal["http", "https"] = Field(default="http", alias="http-scheme") + """ HTTP scheme that should be used while establishing a connection to the Trino cluster """ + + verify: StrictBool = Field(default=True, alias="ssl-verify") + """ Whether the SSL certificate emited by the Trino cluster should be verified or not """ + + extra_credential: Optional[StrictStr] = Field( + default=None, alias="x-trino-extra-credential-header" + ) + """ Specifies the HTTP header X-Trino-Extra-Credential, e.g. user1=pwd1, user2=pwd2 """ + connector: Dict[str, str] """ Trino connector to use as well as potential extra parameters. @@ -59,6 +163,16 @@ class TrinoOfflineStoreConfig(FeastConfigBaseModel): dataset: StrictStr = "feast" """ (optional) Trino Dataset name for temporary tables """ + auth: Optional[AuthConfig] + """ + (optional) Authentication mechanism to use when connecting to Trino. Supported options are: + - kerberos + - basic + - jwt + - oauth2 + - certificate + """ + class TrinoRetrievalJob(RetrievalJob): def __init__( @@ -162,9 +276,6 @@ def pull_latest_from_table_or_query( created_timestamp_column: Optional[str], start_date: datetime, end_date: datetime, - user: Optional[str] = None, - auth: Optional[Authentication] = None, - http_scheme: Optional[str] = None, ) -> TrinoRetrievalJob: assert isinstance(config.offline_store, TrinoOfflineStoreConfig) assert isinstance(data_source, TrinoSource) @@ -181,9 +292,7 @@ def pull_latest_from_table_or_query( timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) - client = _get_trino_client( - config=config, user=user, auth=auth, http_scheme=http_scheme - ) + client = _get_trino_client(config=config) query = f""" SELECT @@ -216,17 +325,12 @@ def get_historical_features( registry: Registry, project: str, full_feature_names: bool = False, - user: Optional[str] = None, - auth: Optional[Authentication] = None, - http_scheme: Optional[str] = None, ) -> TrinoRetrievalJob: assert isinstance(config.offline_store, TrinoOfflineStoreConfig) for fv in feature_views: assert isinstance(fv.batch_source, TrinoSource) - client = _get_trino_client( - config=config, user=user, auth=auth, http_scheme=http_scheme - ) + client = _get_trino_client(config=config) table_reference = _get_table_reference_for_new_entity( catalog=config.offline_store.catalog, @@ -307,17 +411,12 @@ def pull_all_from_table_or_query( timestamp_field: str, start_date: datetime, end_date: datetime, - user: Optional[str] = None, - auth: Optional[Authentication] = None, - http_scheme: Optional[str] = None, ) -> RetrievalJob: assert isinstance(config.offline_store, TrinoOfflineStoreConfig) assert isinstance(data_source, TrinoSource) from_expression = data_source.get_table_query_string() - client = _get_trino_client( - config=config, user=user, auth=auth, http_scheme=http_scheme - ) + client = _get_trino_client(config=config) field_string = ", ".join( join_key_columns + feature_name_columns + [timestamp_field] ) @@ -378,21 +477,22 @@ def _upload_entity_df_and_get_entity_schema( # TODO: Ensure that the table expires after some time -def _get_trino_client( - config: RepoConfig, - user: Optional[str], - auth: Optional[Any], - http_scheme: Optional[str], -) -> Trino: - client = Trino( - user=user, - catalog=config.offline_store.catalog, +def _get_trino_client(config: RepoConfig) -> Trino: + auth = None + if config.offline_store.auth is not None: + auth = config.offline_store.auth.to_trino_auth() + + return Trino( host=config.offline_store.host, port=config.offline_store.port, + user=config.offline_store.user, + catalog=config.offline_store.catalog, + source=config.offline_store.source, + http_scheme=config.offline_store.http_scheme, + verify=config.offline_store.verify, + extra_credential=config.offline_store.extra_credential, auth=auth, - http_scheme=http_scheme, ) - return client def _get_entity_df_event_timestamp_range( diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py index 97c61f78a60..50472407bc6 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_queries.py @@ -1,7 +1,6 @@ from __future__ import annotations import datetime -import os import signal from dataclasses import dataclass from enum import Enum @@ -30,34 +29,27 @@ class QueryStatus(Enum): class Trino: def __init__( self, - host: Optional[str] = None, - port: Optional[int] = None, - user: Optional[str] = None, - catalog: Optional[str] = None, - auth: Optional[Any] = None, - http_scheme: Optional[str] = None, - source: Optional[str] = None, - extra_credential: Optional[str] = None, + host: str, + port: int, + user: str, + catalog: str, + source: Optional[str], + http_scheme: str, + verify: bool, + extra_credential: Optional[str], + auth: Optional[trino.Authentication], ): - self.host = host or os.getenv("TRINO_HOST") - self.port = port or os.getenv("TRINO_PORT") - self.user = user or os.getenv("TRINO_USER") - self.catalog = catalog or os.getenv("TRINO_CATALOG") - self.auth = auth or os.getenv("TRINO_AUTH") - self.http_scheme = http_scheme or os.getenv("TRINO_HTTP_SCHEME") - self.source = source or os.getenv("TRINO_SOURCE") - self.extra_credential = extra_credential or os.getenv("TRINO_EXTRA_CREDENTIAL") + self.host = host + self.port = port + self.user = user + self.catalog = catalog + self.source = source + self.http_scheme = http_scheme + self.verify = verify + self.extra_credential = extra_credential + self.auth = auth self._cursor: Optional[Cursor] = None - if self.host is None: - raise ValueError("TRINO_HOST must be set if not passed in") - if self.port is None: - raise ValueError("TRINO_PORT must be set if not passed in") - if self.user is None: - raise ValueError("TRINO_USER must be set if not passed in") - if self.catalog is None: - raise ValueError("TRINO_CATALOG must be set if not passed in") - def _get_cursor(self) -> Cursor: if self._cursor is None: headers = ( @@ -70,9 +62,10 @@ def _get_cursor(self) -> Cursor: port=self.port, user=self.user, catalog=self.catalog, - auth=self.auth, - http_scheme=self.http_scheme, source=self.source, + http_scheme=self.http_scheme, + verify=self.verify, + auth=self.auth, http_headers=headers, ).cursor() diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py index f09b79069ca..e618e8664ee 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino_source.py @@ -227,10 +227,20 @@ def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: + auth = None + if config.offline_store.auth is not None: + auth = config.offline_store.auth.to_trino_auth() + client = Trino( catalog=config.offline_store.catalog, host=config.offline_store.host, port=config.offline_store.port, + user=config.offline_store.user, + source=config.offline_store.source, + http_scheme=config.offline_store.http_scheme, + verify=config.offline_store.verify, + extra_credential=config.offline_store.extra_credential, + auth=auth, ) if self.table: table_schema = client.execute_query( From a8aeee9981fe75ffdc234d63c58cc99c5d3013db Mon Sep 17 00:00:00 2001 From: feast-ci-bot Date: Thu, 7 Sep 2023 06:53:29 +0000 Subject: [PATCH 18/18] chore(release): release 0.34.0 # [0.34.0](https://github.com/feast-dev/feast/compare/v0.33.0...v0.34.0) (2023-09-07) ### Bug Fixes * Add NUMERIC to bq_to_feast type map ([#3719](https://github.com/feast-dev/feast/issues/3719)) ([6474b4b](https://github.com/feast-dev/feast/commit/6474b4b0169dc9b3df8e8daecded2b1fad5ead58)) * Fix python unit tests ([#3734](https://github.com/feast-dev/feast/issues/3734)) ([e81684d](https://github.com/feast-dev/feast/commit/e81684d4f7916c986fa8e6cf06c2918951469799)) * Handle unknown postgres source types gracefully ([#3634](https://github.com/feast-dev/feast/issues/3634)) ([d7041f4](https://github.com/feast-dev/feast/commit/d7041f4cce813d349e9016da55d65a65c1ec2355)) * Pin protobuf version to avoid seg fault on some machines ([028cc20](https://github.com/feast-dev/feast/commit/028cc20a28118bd31deca8965782d5ad25f74300)) * Remove unwanted excessive splitting of gcs path, so expected gcs parquet paths are returned from BigQueryRetrievalJob.to_remote_storage() ([#3730](https://github.com/feast-dev/feast/issues/3730)) ([f2c5988](https://github.com/feast-dev/feast/commit/f2c59885e31f3f238dbd9c13cd1ba168e3233a9d)) * Run store.plan() only when need it. ([#3708](https://github.com/feast-dev/feast/issues/3708)) ([7bc7c47](https://github.com/feast-dev/feast/commit/7bc7c47b4507310850474290131c03fb6d480834)) * Saved datasets no longer break CLI registry-dump command ([#3717](https://github.com/feast-dev/feast/issues/3717)) ([f28ccc2](https://github.com/feast-dev/feast/commit/f28ccc2b8f42bcca943d498ad583337d4cd70383)) * Update py3.8 ci requirements for cython 3.0 release ([#3735](https://github.com/feast-dev/feast/issues/3735)) ([1695c13](https://github.com/feast-dev/feast/commit/1695c13fa8f48fdc2b5e627837043c5eea0914a9)) ### Features * Enhance customization of Trino connections when using Trino-based Offline Stores ([#3699](https://github.com/feast-dev/feast/issues/3699)) ([ed7535e](https://github.com/feast-dev/feast/commit/ed7535e23d490249ca7d224fb88e53b98d496ec0)) * Implement gRPC server to ingest streaming features ([#3687](https://github.com/feast-dev/feast/issues/3687)) ([a3fcd1f](https://github.com/feast-dev/feast/commit/a3fcd1f369bdf07174b5ecf2a49ca9864cf145d4)) --- CHANGELOG.md | 20 +++++++++++++++++++ infra/charts/feast-feature-server/Chart.yaml | 2 +- infra/charts/feast-feature-server/README.md | 4 ++-- infra/charts/feast-feature-server/values.yaml | 2 +- infra/charts/feast/Chart.yaml | 2 +- infra/charts/feast/README.md | 6 +++--- .../feast/charts/feature-server/Chart.yaml | 4 ++-- .../feast/charts/feature-server/README.md | 4 ++-- .../feast/charts/feature-server/values.yaml | 2 +- .../charts/transformation-service/Chart.yaml | 4 ++-- .../charts/transformation-service/README.md | 4 ++-- .../charts/transformation-service/values.yaml | 2 +- infra/charts/feast/requirements.yaml | 4 ++-- java/pom.xml | 2 +- sdk/python/feast/ui/package.json | 2 +- sdk/python/feast/ui/yarn.lock | 8 ++++---- ui/package.json | 2 +- 17 files changed, 47 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 585be439bac..f6e5a430f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +# [0.34.0](https://github.com/feast-dev/feast/compare/v0.33.0...v0.34.0) (2023-09-07) + + +### Bug Fixes + +* Add NUMERIC to bq_to_feast type map ([#3719](https://github.com/feast-dev/feast/issues/3719)) ([6474b4b](https://github.com/feast-dev/feast/commit/6474b4b0169dc9b3df8e8daecded2b1fad5ead58)) +* Fix python unit tests ([#3734](https://github.com/feast-dev/feast/issues/3734)) ([e81684d](https://github.com/feast-dev/feast/commit/e81684d4f7916c986fa8e6cf06c2918951469799)) +* Handle unknown postgres source types gracefully ([#3634](https://github.com/feast-dev/feast/issues/3634)) ([d7041f4](https://github.com/feast-dev/feast/commit/d7041f4cce813d349e9016da55d65a65c1ec2355)) +* Pin protobuf version to avoid seg fault on some machines ([028cc20](https://github.com/feast-dev/feast/commit/028cc20a28118bd31deca8965782d5ad25f74300)) +* Remove unwanted excessive splitting of gcs path, so expected gcs parquet paths are returned from BigQueryRetrievalJob.to_remote_storage() ([#3730](https://github.com/feast-dev/feast/issues/3730)) ([f2c5988](https://github.com/feast-dev/feast/commit/f2c59885e31f3f238dbd9c13cd1ba168e3233a9d)) +* Run store.plan() only when need it. ([#3708](https://github.com/feast-dev/feast/issues/3708)) ([7bc7c47](https://github.com/feast-dev/feast/commit/7bc7c47b4507310850474290131c03fb6d480834)) +* Saved datasets no longer break CLI registry-dump command ([#3717](https://github.com/feast-dev/feast/issues/3717)) ([f28ccc2](https://github.com/feast-dev/feast/commit/f28ccc2b8f42bcca943d498ad583337d4cd70383)) +* Update py3.8 ci requirements for cython 3.0 release ([#3735](https://github.com/feast-dev/feast/issues/3735)) ([1695c13](https://github.com/feast-dev/feast/commit/1695c13fa8f48fdc2b5e627837043c5eea0914a9)) + + +### Features + +* Enhance customization of Trino connections when using Trino-based Offline Stores ([#3699](https://github.com/feast-dev/feast/issues/3699)) ([ed7535e](https://github.com/feast-dev/feast/commit/ed7535e23d490249ca7d224fb88e53b98d496ec0)) +* Implement gRPC server to ingest streaming features ([#3687](https://github.com/feast-dev/feast/issues/3687)) ([a3fcd1f](https://github.com/feast-dev/feast/commit/a3fcd1f369bdf07174b5ecf2a49ca9864cf145d4)) + # [0.33.0](https://github.com/feast-dev/feast/compare/v0.32.0...v0.33.0) (2023-08-14) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 3ab493fc5fb..6631d377844 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.33.0 +version: 0.34.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index e175f78510a..ad88d082178 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.33.0` +Current chart version is `0.34.0` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.33.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.34.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index e9d09796a31..d46f1b685b3 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.33.0 + tag: 0.34.0 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index b16165808f8..e0f530e05ee 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.33.0 +version: 0.34.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 846e8402361..fff6d0261bd 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.33.0` +Feature store for machine learning Current chart version is `0.34.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.33.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.33.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.34.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.34.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 931c34f7c94..bfb33b6140f 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.33.0 -appVersion: v0.33.0 +version: 0.34.0 +appVersion: v0.34.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 653b9dfb0c4..f768a46cd53 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.33.0](https://img.shields.io/badge/Version-0.33.0-informational?style=flat-square) ![AppVersion: v0.33.0](https://img.shields.io/badge/AppVersion-v0.33.0-informational?style=flat-square) +![Version: 0.34.0](https://img.shields.io/badge/Version-0.34.0-informational?style=flat-square) ![AppVersion: v0.34.0](https://img.shields.io/badge/AppVersion-v0.34.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.33.0"` | Image tag | +| image.tag | string | `"0.34.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 2403e796458..24b8da1e39a 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.33.0 + tag: 0.34.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 7a4e0fd7d17..5d8f157a48f 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.33.0 -appVersion: v0.33.0 +version: 0.34.0 +appVersion: v0.34.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 0d217edf046..cf8c7eaae87 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.33.0](https://img.shields.io/badge/Version-0.33.0-informational?style=flat-square) ![AppVersion: v0.33.0](https://img.shields.io/badge/AppVersion-v0.33.0-informational?style=flat-square) +![Version: 0.34.0](https://img.shields.io/badge/Version-0.34.0-informational?style=flat-square) ![AppVersion: v0.34.0](https://img.shields.io/badge/AppVersion-v0.34.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.33.0"` | Image tag | +| image.tag | string | `"0.34.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index b7fdcb66590..6af9e569ea9 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.33.0 + tag: 0.34.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 5f855ef5278..b3236b03228 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.33.0 + version: 0.34.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.33.0 + version: 0.34.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/java/pom.xml b/java/pom.xml index 0aa87a6feab..245d2d7cc6e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.33.0 + 0.34.0 https://github.com/feast-dev/feast UTF-8 diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 83722808820..80956440abf 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.33.0", + "@feast-dev/feast-ui": "0.34.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 28bfee06d1e..6fb431576ae 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1452,10 +1452,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.33.0": - version "0.33.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.33.0.tgz#4b8f8c5376103cac1ae0f25d6c9d359e4022707a" - integrity sha512-w+wa+YpFOIbxr4zTmGPuPITQhWfYIWjv7OmL7r/9yLbtQzp/6sKKChGYZwxi6pUu1ECsRc47uuSyhYllav3InQ== +"@feast-dev/feast-ui@0.34.0": + version "0.34.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.34.0.tgz#21799c119da936814ff47e1d1d297258ecfda098" + integrity sha512-ErrnmaFPMgnmnTpBpn7T0oiC3cA+atE5yKBmoMFXBsK6aplu18aF53QBX4JJFKMKF+A2BcO37veeXm1k01SJQQ== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" diff --git a/ui/package.json b/ui/package.json index b81276d8257..dc16a1e7a6a 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.33.0", + "version": "0.34.0", "private": false, "files": [ "dist"