Skip to content

Commit b9aabbd

Browse files
szabofereeszaboferee
authored
fix: Azure blob storage support in Java feature server (#2319) (#4014)
- Add azure blob storage support in java feature server - Fix S3 integration test to work without a real AWS account - Add GCS mock to integration tests to be able to run them without a real google cloud account - Adding dependency management in maven for libraries with older incompatible versions as transitive dependencies Signed-off-by: Ferenc Szabó <szaboferee@gmail.com>
1 parent 3980e0c commit b9aabbd

File tree

10 files changed

+335
-52
lines changed

10 files changed

+335
-52
lines changed

‎java/CONTRIBUTING.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ Automatically format the code to conform the style guide by:
5050

5151
```sh
5252
# formats all code in the feast-java repository
53-
mvn spotless:apply
53+
make format-java
5454
```
5555

5656
> If you're using IntelliJ, you can import these [code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml)
@@ -66,7 +66,7 @@ Run all Unit tests:
6666
make test-java
6767
```
6868

69-
Run all Integration tests (note: this also runs GCS + S3 based tests which should fail):
69+
Run all Integration tests:
7070
```
7171
make test-java-integration
7272
```

‎java/pom.xml‎

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@
6868
<google.auth.library.oauth2.http.version>0.21.0</google.auth.library.oauth2.http.version>
6969
<auto.value.version>1.6.6</auto.value.version>
7070
<guava.version>30.1-jre</guava.version>
71+
<reactor.version>3.4.34</reactor.version>
72+
<netty.version>4.1.101.Final</netty.version>
7173

7274
<license.content><![CDATA[
7375
/*
@@ -186,6 +188,49 @@
186188
<version>${javax.validation.version}</version>
187189
</dependency>
188190

191+
<dependency>
192+
<groupId>com.fasterxml.jackson.core</groupId>
193+
<artifactId>jackson-core</artifactId>
194+
<version>${jackson.version}</version>
195+
</dependency>
196+
<dependency>
197+
<groupId>com.fasterxml.jackson.core</groupId>
198+
<artifactId>jackson-databind</artifactId>
199+
<version>${jackson.version}</version>
200+
</dependency>
201+
<dependency>
202+
<groupId>com.fasterxml.jackson.core</groupId>
203+
<artifactId>jackson-annotations</artifactId>
204+
<version>${jackson.version}</version>
205+
</dependency>
206+
207+
<dependency>
208+
<groupId>io.netty</groupId>
209+
<artifactId>netty-common</artifactId>
210+
<version>${netty.version}</version>
211+
</dependency>
212+
<dependency>
213+
<groupId>io.netty</groupId>
214+
<artifactId>netty-buffer</artifactId>
215+
<version>${netty.version}</version>
216+
</dependency>
217+
<dependency>
218+
<groupId>io.netty</groupId>
219+
<artifactId>netty-handler</artifactId>
220+
<version>${netty.version}</version>
221+
</dependency>
222+
<dependency>
223+
<groupId>io.netty</groupId>
224+
<artifactId>netty-transport</artifactId>
225+
<version>${netty.version}</version>
226+
</dependency>
227+
228+
<dependency>
229+
<groupId>io.projectreactor</groupId>
230+
<artifactId>reactor-core</artifactId>
231+
<version>${reactor.version}</version>
232+
</dependency>
233+
189234
<dependency>
190235
<groupId>org.junit.platform</groupId>
191236
<artifactId>junit-platform-engine</artifactId>
@@ -246,7 +291,7 @@
246291
<configuration>
247292
<java>
248293
<licenseHeader>
249-
<content>${license.content}</content>
294+
<content>${license.content}</content>
250295
</licenseHeader>
251296
<googleJavaFormat>
252297
<version>1.7</version>
@@ -264,15 +309,15 @@
264309
</scala>
265310
</configuration>
266311
<executions>
267-
<!-- Move check to fail faster, but after compilation. Default is verify phase -->
268-
<execution>
269-
<id>spotless-check</id>
270-
<phase>process-test-classes</phase>
271-
<goals>
272-
<goal>check</goal>
273-
</goals>
274-
</execution>
275-
</executions>
312+
<!-- Move check to fail faster, but after compilation. Default is verify phase -->
313+
<execution>
314+
<id>spotless-check</id>
315+
<phase>process-test-classes</phase>
316+
<goals>
317+
<goal>check</goal>
318+
</goals>
319+
</execution>
320+
</executions>
276321
</plugin>
277322
<plugin>
278323
<groupId>org.apache.maven.plugins</groupId>

‎java/serving/.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,7 @@ feast-serving.jar
3434
/.nb-gradle/
3535

3636
## Feast Temporary Files ##
37-
/temp/
37+
/temp/
38+
39+
## Generated test data ##
40+
**/*.parquet

‎java/serving/pom.xml‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
~
1717
-->
1818
<project xmlns="http://maven.apache.org/POM/4.0.0"
19-
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
2121
<modelVersion>4.0.0</modelVersion>
2222

2323
<parent>
@@ -121,6 +121,19 @@
121121
<version>5.0.1</version>
122122
</dependency>
123123

124+
<!-- azure blob storage -->
125+
<!-- https://mvnrepository.com/artifact/com.azure/azure-storage-blob -->
126+
<dependency>
127+
<groupId>com.azure</groupId>
128+
<artifactId>azure-storage-blob</artifactId>
129+
<version>12.25.2</version>
130+
</dependency>
131+
<dependency>
132+
<groupId>com.azure</groupId>
133+
<artifactId>azure-identity</artifactId>
134+
<version>1.11.3</version>
135+
</dependency>
136+
124137
<!-- TODO: SLF4J is being used via Lombok, but also jog4j - pick one -->
125138
<dependency>
126139
<groupId>org.slf4j</groupId>
@@ -356,11 +369,11 @@
356369
<version>2.7.4</version>
357370
<scope>test</scope>
358371
</dependency>
359-
<dependency>
360-
<groupId>io.lettuce</groupId>
361-
<artifactId>lettuce-core</artifactId>
362-
<version>6.0.2.RELEASE</version>
363-
</dependency>
372+
<dependency>
373+
<groupId>io.lettuce</groupId>
374+
<artifactId>lettuce-core</artifactId>
375+
<version>6.0.2.RELEASE</version>
376+
</dependency>
364377
<dependency>
365378
<groupId>org.apache.commons</groupId>
366379
<artifactId>commons-lang3</artifactId>

‎java/serving/src/main/java/feast/serving/registry/AzureRegistryFile.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2021 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.serving.registry;
18+
19+
import com.azure.storage.blob.BlobClient;
20+
import com.azure.storage.blob.BlobServiceClient;
21+
import com.google.protobuf.InvalidProtocolBufferException;
22+
import feast.proto.core.RegistryProto;
23+
import java.util.Objects;
24+
import java.util.Optional;
25+
26+
public class AzureRegistryFile implements RegistryFile {
27+
private final BlobClient blobClient;
28+
private String lastKnownETag;
29+
30+
public AzureRegistryFile(BlobServiceClient blobServiceClient, String url) {
31+
String[] split = url.replace("az://", "").split("/");
32+
String objectPath = String.join("/", java.util.Arrays.copyOfRange(split, 1, split.length));
33+
this.blobClient = blobServiceClient.getBlobContainerClient(split[0]).getBlobClient(objectPath);
34+
}
35+
36+
@Override
37+
public RegistryProto.Registry getContent() {
38+
try {
39+
return RegistryProto.Registry.parseFrom(blobClient.downloadContent().toBytes());
40+
} catch (InvalidProtocolBufferException e) {
41+
throw new RuntimeException(
42+
String.format(
43+
"Couldn't read remote registry: %s. Error: %s",
44+
blobClient.getBlobUrl(), e.getMessage()));
45+
}
46+
}
47+
48+
@Override
49+
public Optional<RegistryProto.Registry> getContentIfModified() {
50+
String eTag = blobClient.getProperties().getETag();
51+
if (Objects.equals(eTag, this.lastKnownETag)) {
52+
return Optional.empty();
53+
} else this.lastKnownETag = eTag;
54+
55+
return Optional.of(getContent());
56+
}
57+
}

‎java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ public static class FeastProperties {
9595
private String gcpProject;
9696
private String awsRegion;
9797
private String transformationServiceEndpoint;
98+
private String azureStorageAccount;
9899

99100
public String getRegistry() {
100101
return registry;
@@ -205,6 +206,14 @@ public String getTransformationServiceEndpoint() {
205206
public void setTransformationServiceEndpoint(String transformationServiceEndpoint) {
206207
this.transformationServiceEndpoint = transformationServiceEndpoint;
207208
}
209+
210+
public String getAzureStorageAccount() {
211+
return azureStorageAccount;
212+
}
213+
214+
public void setAzureStorageAccount(String azureStorageAccount) {
215+
this.azureStorageAccount = azureStorageAccount;
216+
}
208217
}
209218

210219
/** Store configuration class for database that this Feast Serving uses. */

‎java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818

1919
import com.amazonaws.services.s3.AmazonS3;
2020
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
21+
import com.azure.identity.DefaultAzureCredentialBuilder;
22+
import com.azure.storage.blob.BlobServiceClient;
23+
import com.azure.storage.blob.BlobServiceClientBuilder;
2124
import com.google.cloud.storage.Storage;
2225
import com.google.cloud.storage.StorageOptions;
2326
import com.google.inject.AbstractModule;
@@ -43,11 +46,27 @@ public AmazonS3 awsStorage(ApplicationProperties applicationProperties) {
4346
.build();
4447
}
4548

49+
@Provides
50+
public BlobServiceClient azureStorage(ApplicationProperties applicationProperties) {
51+
52+
BlobServiceClient blobServiceClient =
53+
new BlobServiceClientBuilder()
54+
.endpoint(
55+
String.format(
56+
"https://%s.blob.core.windows.net",
57+
applicationProperties.getFeast().getAzureStorageAccount()))
58+
.credential(new DefaultAzureCredentialBuilder().build())
59+
.buildClient();
60+
61+
return blobServiceClient;
62+
}
63+
4664
@Provides
4765
RegistryFile registryFile(
4866
ApplicationProperties applicationProperties,
4967
Provider<Storage> storageProvider,
50-
Provider<AmazonS3> amazonS3Provider) {
68+
Provider<AmazonS3> amazonS3Provider,
69+
Provider<BlobServiceClient> azureProvider) {
5170

5271
String registryPath = applicationProperties.getFeast().getRegistry();
5372
Optional<String> scheme = Optional.ofNullable(URI.create(registryPath).getScheme());
@@ -57,6 +76,8 @@ RegistryFile registryFile(
5776
return new GSRegistryFile(storageProvider.get(), registryPath);
5877
case "s3":
5978
return new S3RegistryFile(amazonS3Provider.get(), registryPath);
79+
case "az":
80+
return new AzureRegistryFile(azureProvider.get(), registryPath);
6081
case "":
6182
case "file":
6283
return new LocalRegistryFile(registryPath);

0 commit comments

Comments
 (0)