@@ -33,7 +33,7 @@ Feast allows ML platform teams to: | |||
| 33 | 33 | ||
| 34 | 34 | * **Make features consistently available for training and serving** by managing an _offline store_ (to process historical data for scale-out batch scoring or model training), a low-latency _online store_ (to power real-time prediction)_,_ and a battle-tested _feature server_ (to serve pre-computed features online). | |
| 35 | 35 | * **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensure that future feature values do not leak to models during training. | |
| 36 | - * **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to real-time models, and from one data infra system to another. | ||
| 36 | + * **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to realtime models, and from one data infra system to another. | ||
| 37 | 37 | ||
| 38 | 38 | Please see our [documentation](https://docs.feast.dev/) for more information about the project. | |
| 39 | 39 | ||
@@ -7,8 +7,7 @@ | |||
| 7 | 7 | from pytz import FixedOffset, timezone, utc | |
| 8 | 8 | from random import randint | |
| 9 | 9 | from enum import Enum | |
| 10 | - from sqlalchemy import create_engine, DateTime | ||
| 11 | - from datetime import datetime | ||
| 10 | + from sqlalchemy import DateTime | ||
| 12 | 11 | ||
| 13 | 12 | DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL = "event_timestamp" | |
| 14 | 13 | ||
@@ -65,7 +64,8 @@ def create_orders_df( | |||
| 65 | 64 | ) | |
| 66 | 65 | ] | |
| 67 | 66 | df.sort_values( | |
| 68 | - by=["e_ts", "order_id", "driver_id", "customer_id"], inplace=True, | ||
| 67 | + by=["e_ts", "order_id", "driver_id", "customer_id"], | ||
| 68 | + inplace=True, | ||
| 69 | 69 | ) | |
| 70 | 70 | else: | |
| 71 | 71 | df[DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL] = [ | |
@@ -208,9 +208,7 @@ def create_customer_daily_profile_df(customers, start_date, end_date) -> pd.Data | |||
| 208 | 208 | ||
| 209 | 209 | def generate_entities(date, n_customers, n_drivers, order_count): | |
| 210 | 210 | end_date = date | |
| 211 | - before_start_date = end_date - timedelta(days=365) | ||
| 212 | 211 | start_date = end_date - timedelta(days=7) | |
| 213 | - after_end_date = end_date + timedelta(days=365) | ||
| 214 | 212 | customer_entities = [20000 + c_id for c_id in range(n_customers)] | |
| 215 | 213 | driver_entities = [50000 + d_id for d_id in range(n_drivers)] | |
| 216 | 214 | orders_df = create_orders_df( | |
@@ -225,7 +223,7 @@ def generate_entities(date, n_customers, n_drivers, order_count): | |||
| 225 | 223 | ||
| 226 | 224 | ||
| 227 | 225 | def save_df_to_csv(df, table_name, dtype): | |
| 228 | - df.to_csv(table_name+".csv", index=False) | ||
| 226 | + df.to_csv(table_name + ".csv", index=False) | ||
| 229 | 227 | ||
| 230 | 228 | ||
| 231 | 229 | if __name__ == "__main__": | |
@@ -247,7 +245,6 @@ def save_df_to_csv(df, table_name, dtype): | |||
| 247 | 245 | ||
| 248 | 246 | print(drivers_df.head()) | |
| 249 | 247 | ||
| 250 | - | ||
| 251 | 248 | orders_table = "orders" | |
| 252 | 249 | driver_hourly_table = "driver_hourly" | |
| 253 | 250 | customer_profile_table = "customer_profile" | |
@@ -257,4 +254,4 @@ def save_df_to_csv(df, table_name, dtype): | |||
| 257 | 254 | print("uploading drivers") | |
| 258 | 255 | save_df_to_csv(drivers_df, driver_hourly_table, dtype={"datetime": DateTime()}) | |
| 259 | 256 | print("uploading customers") | |
| 260 | - save_df_to_csv(customer_df, customer_profile_table, dtype={"datetime": DateTime()}) | ||
| 257 | + save_df_to_csv(customer_df, customer_profile_table, dtype={"datetime": DateTime()}) | ||
@@ -11,7 +11,7 @@ | |||
| 11 | 11 | from feast.infra.offline_stores.contrib.mssql_offline_store.mssql import ( | |
| 12 | 12 | MsSqlServerOfflineStoreConfig, | |
| 13 | 13 | ) | |
| 14 | - from feast.infra.online_stores.redis import RedisOnlineStoreConfig, RedisOnlineStore | ||
| 14 | + from feast.infra.online_stores.redis import RedisOnlineStoreConfig | ||
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | 17 | def init(): | |
@@ -14,7 +14,10 @@ | |||
| 14 | 14 | timestamp_field="event_timestamp", | |
| 15 | 15 | created_timestamp_column="created", | |
| 16 | 16 | ) | |
| 17 | - driver = Entity(name="driver_id", description="driver id",) | ||
| 17 | + driver = Entity( | ||
| 18 | + name="driver_id", | ||
| 19 | + description="driver id", | ||
| 20 | + ) | ||
| 18 | 21 | driver_hourly_stats_view = FeatureView( | |
| 19 | 22 | name="driver_hourly_stats", | |
| 20 | 23 | entities=[driver], | |
@@ -58,4 +61,3 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: | |||
| 58 | 61 | df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] | |
| 59 | 62 | df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] | |
| 60 | 63 | return df | |
| 61 | - | ||
@@ -20,9 +20,13 @@ def fetch_java(): | |||
| 20 | 20 | ||
| 21 | 21 | print( | |
| 22 | 22 | stub.GetOnlineFeatures( | |
| 23 | - GetOnlineFeaturesRequest(features=feature_refs, entities=entity_rows,) | ||
| 23 | + GetOnlineFeaturesRequest( | ||
| 24 | + features=feature_refs, | ||
| 25 | + entities=entity_rows, | ||
| 26 | + ) | ||
| 24 | 27 | ) | |
| 25 | 28 | ) | |
| 26 | 29 | ||
| 30 | + | ||
| 27 | 31 | if __name__ == "__main__": | |
| 28 | 32 | fetch_java() | |
@@ -15,7 +15,7 @@ def run_demo(): | |||
| 15 | 15 | }, | |
| 16 | 16 | { | |
| 17 | 17 | "driver_id": 1002, | |
| 18 | - } | ||
| 18 | + }, | ||
| 19 | 19 | ], | |
| 20 | 20 | ).to_dict() | |
| 21 | 21 | for key, value in sorted(features.items()): | |
@@ -1,12 +1,18 @@ | |||
| 1 | 1 | import subprocess | |
| 2 | 2 | ||
| 3 | - def port_forward(service, external_port, local_port=80) : | ||
| 4 | - """ | ||
| 5 | - Run a background process to forward port 80 of the given `service` service to the given `external_port` port. | ||
| 6 | 3 | ||
| 7 | - Returns: the process instance | ||
| 8 | - """ | ||
| 9 | - command = ["kubectl", "port-forward", f"service/{service}", f"{external_port}:{local_port}"] | ||
| 10 | - process = subprocess.Popen(command) | ||
| 11 | - print(f"Port-forwarding {service} with process ID: {process.pid}") | ||
| 12 | - return process | ||
| 4 | + def port_forward(service, external_port, local_port=80): | ||
| 5 | + """ | ||
| 6 | + Run a background process to forward port 80 of the given `service` service to the given `external_port` port. | ||
| 7 | + | ||
| 8 | + Returns: the process instance | ||
| 9 | + """ | ||
| 10 | + command = [ | ||
| 11 | + "kubectl", | ||
| 12 | + "port-forward", | ||
| 13 | + f"service/{service}", | ||
| 14 | + f"{external_port}:{local_port}", | ||
| 15 | + ] | ||
| 16 | + process = subprocess.Popen(command) | ||
| 17 | + print(f"Port-forwarding {service} with process ID: {process.pid}") | ||
| 18 | + return process | ||
@@ -32,39 +32,71 @@ def generate_sample_data(): | |||
| 32 | 32 | ||
| 33 | 33 | # Sample product data | |
| 34 | 34 | products = [ | |
| 35 | - {"id": 1, "name": "Smartphone", | ||
| 36 | - "description": "A high-end smartphone with advanced camera features and long battery life."}, | ||
| 37 | - {"id": 2, "name": "Laptop", | ||
| 38 | - "description": "Powerful laptop with fast processor and high-resolution display for professional use."}, | ||
| 39 | - {"id": 3, "name": "Headphones", | ||
| 40 | - "description": "Wireless noise-cancelling headphones with premium sound quality."}, | ||
| 41 | - {"id": 4, "name": "Smartwatch", | ||
| 42 | - "description": "Fitness tracking smartwatch with heart rate monitoring and sleep analysis."}, | ||
| 43 | - {"id": 5, "name": "Tablet", | ||
| 44 | - "description": "Lightweight tablet with vibrant display perfect for reading and browsing."}, | ||
| 45 | - {"id": 6, "name": "Camera", | ||
| 46 | - "description": "Professional digital camera with high-resolution sensor and interchangeable lenses."}, | ||
| 47 | - {"id": 7, "name": "Speaker", | ||
| 48 | - "description": "Bluetooth speaker with rich bass and long battery life for outdoor use."}, | ||
| 49 | - {"id": 8, "name": "Gaming Console", | ||
| 50 | - "description": "Next-generation gaming console with 4K graphics and fast loading times."}, | ||
| 51 | - {"id": 9, "name": "E-reader", | ||
| 52 | - "description": "E-ink display reader with backlight for comfortable reading in any lighting condition."}, | ||
| 53 | - {"id": 10, "name": "Smart TV", | ||
| 54 | - "description": "4K smart television with built-in streaming apps and voice control."} | ||
| 35 | + { | ||
| 36 | + "id": 1, | ||
| 37 | + "name": "Smartphone", | ||
| 38 | + "description": "A high-end smartphone with advanced camera features and long battery life.", | ||
| 39 | + }, | ||
| 40 | + { | ||
| 41 | + "id": 2, | ||
| 42 | + "name": "Laptop", | ||
| 43 | + "description": "Powerful laptop with fast processor and high-resolution display for professional use.", | ||
| 44 | + }, | ||
| 45 | + { | ||
| 46 | + "id": 3, | ||
| 47 | + "name": "Headphones", | ||
| 48 | + "description": "Wireless noise-cancelling headphones with premium sound quality.", | ||
| 49 | + }, | ||
| 50 | + { | ||
| 51 | + "id": 4, | ||
| 52 | + "name": "Smartwatch", | ||
| 53 | + "description": "Fitness tracking smartwatch with heart rate monitoring and sleep analysis.", | ||
| 54 | + }, | ||
| 55 | + { | ||
| 56 | + "id": 5, | ||
| 57 | + "name": "Tablet", | ||
| 58 | + "description": "Lightweight tablet with vibrant display perfect for reading and browsing.", | ||
| 59 | + }, | ||
| 60 | + { | ||
| 61 | + "id": 6, | ||
| 62 | + "name": "Camera", | ||
| 63 | + "description": "Professional digital camera with high-resolution sensor and interchangeable lenses.", | ||
| 64 | + }, | ||
| 65 | + { | ||
| 66 | + "id": 7, | ||
| 67 | + "name": "Speaker", | ||
| 68 | + "description": "Bluetooth speaker with rich bass and long battery life for outdoor use.", | ||
| 69 | + }, | ||
| 70 | + { | ||
| 71 | + "id": 8, | ||
| 72 | + "name": "Gaming Console", | ||
| 73 | + "description": "Next-generation gaming console with 4K graphics and fast loading times.", | ||
| 74 | + }, | ||
| 75 | + { | ||
| 76 | + "id": 9, | ||
| 77 | + "name": "E-reader", | ||
| 78 | + "description": "E-ink display reader with backlight for comfortable reading in any lighting condition.", | ||
| 79 | + }, | ||
| 80 | + { | ||
| 81 | + "id": 10, | ||
| 82 | + "name": "Smart TV", | ||
| 83 | + "description": "4K smart television with built-in streaming apps and voice control.", | ||
| 84 | + }, | ||
| 55 | 85 | ] | |
| 56 | 86 | ||
| 57 | 87 | # Create DataFrame | |
| 58 | 88 | df = pd.DataFrame(products) | |
| 59 | 89 | ||
| 60 | 90 | # Generate embeddings using sentence-transformers | |
| 61 | - model = SentenceTransformer('all-MiniLM-L6-v2') # Small, fast model with 384-dim embeddings | ||
| 62 | - embeddings = model.encode(df['description'].tolist()) | ||
| 91 | + model = SentenceTransformer( | ||
| 92 | + "all-MiniLM-L6-v2" | ||
| 93 | + ) # Small, fast model with 384-dim embeddings | ||
| 94 | + embeddings = model.encode(df["description"].tolist()) | ||
| 63 | 95 | ||
| 64 | 96 | # Add embeddings and timestamp to DataFrame | |
| 65 | - df['embedding'] = embeddings.tolist() | ||
| 66 | - df['event_timestamp'] = datetime.now() - timedelta(days=1) | ||
| 67 | - df['created_timestamp'] = datetime.now() - timedelta(days=1) | ||
| 97 | + df["embedding"] = embeddings.tolist() | ||
| 98 | + df["event_timestamp"] = datetime.now() - timedelta(days=1) | ||
| 99 | + df["created_timestamp"] = datetime.now() - timedelta(days=1) | ||
| 68 | 100 | ||
| 69 | 101 | # Save to parquet file | |
| 70 | 102 | parquet_path = "data/sample_data.parquet" | |
@@ -135,16 +167,20 @@ def perform_similarity_search(store, query_text: str, top_k: int = 3): | |||
| 135 | 167 | print(f"\nPerforming similarity search for: '{query_text}'") | |
| 136 | 168 | ||
| 137 | 169 | # Generate embedding for query text | |
| 138 | - model = SentenceTransformer('all-MiniLM-L6-v2') | ||
| 170 | + model = SentenceTransformer("all-MiniLM-L6-v2") | ||
| 139 | 171 | query_embedding = model.encode(query_text).tolist() | |
| 140 | 172 | ||
| 141 | 173 | # Perform similarity search using vector embeddings with version 2 API | |
| 142 | 174 | try: | |
| 143 | 175 | results = store.retrieve_online_documents_v2( | |
| 144 | - features=["product_embeddings:embedding", "product_embeddings:name", "product_embeddings:description"], | ||
| 176 | + features=[ | ||
| 177 | + "product_embeddings:embedding", | ||
| 178 | + "product_embeddings:name", | ||
| 179 | + "product_embeddings:description", | ||
| 180 | + ], | ||
| 145 | 181 | query=query_embedding, | |
| 146 | 182 | top_k=top_k, | |
| 147 | - distance_metric="L2" | ||
| 183 | + distance_metric="L2", | ||
| 148 | 184 | ).to_df() | |
| 149 | 185 | ||
| 150 | 186 | # Print results | |
@@ -184,7 +220,9 @@ def main(): | |||
| 184 | 220 | perform_similarity_search(store, "portable computing device for work", top_k=3) | |
| 185 | 221 | ||
| 186 | 222 | print("\n=== Tutorial Complete ===") | |
| 187 | - print("You've successfully set up Milvus with Feast and performed vector similarity searches!") | ||
| 223 | + print( | ||
| 224 | + "You've successfully set up Milvus with Feast and performed vector similarity searches!" | ||
| 225 | + ) | ||
| 188 | 226 | ||
| 189 | 227 | ||
| 190 | 228 | if __name__ == "__main__": | |
0 commit comments