From a264f7d029ca618d41c598aaf6770ef9139a38b9 Mon Sep 17 00:00:00 2001 From: CaymanWilliams Date: Fri, 21 Jun 2024 08:25:47 -0600 Subject: [PATCH 1/3] small model changes --- databricks/sdk/service/compute.py | 29 +++++++++++++++++++++++++++++ databricks/sdk/service/jobs.py | 15 ++++++++------- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/databricks/sdk/service/compute.py b/databricks/sdk/service/compute.py index db5f550e3..a744b66ac 100755 --- a/databricks/sdk/service/compute.py +++ b/databricks/sdk/service/compute.py @@ -784,12 +784,24 @@ class ClusterDetails: - Name: """ + disk_spec: Optional[dict] = None + """An APC attribute only. This field is missing in the API docs and the unforked databricks + sdk so it needed to be added here""" + docker_image: Optional[DockerImage] = None driver: Optional[SparkNode] = None """Node on which the Spark driver resides. The driver node contains the Spark master and the application that manages the per-notebook Spark REPLs.""" + driver_healthy: Optional[bool] = None + """An APC attribute only. This field is missing in the API docs and the unforked databricks + sdk so it needed to be added here""" + + driver_instance_source: Optional[dict] = None + """An APC attribute only. This field is missing in the API docs and the unforked databricks + sdk so it needed to be added here""" + driver_instance_pool_id: Optional[str] = None """The optional ID of the instance pool for the driver of the cluster belongs. The pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not assigned.""" @@ -798,6 +810,10 @@ class ClusterDetails: """The node type of the Spark driver. Note that this field is optional; if unset, the driver node type will be set as the same value as `node_type_id` defined above.""" + effective_spark_version: Optional[str] = None + """An APC attribute only. This field is missing in the API docs and the unforked databricks + sdk so it needed to be added here""" + enable_elastic_disk: Optional[bool] = None """Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk space when its Spark workers are running low on disk space. This feature requires specific AWS @@ -818,13 +834,25 @@ class ClusterDetails: scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `//init_scripts`.""" + init_scripts_safe_mode: Optional[int] = None + """An APC attribute only. This field is missing in the API docs and the unforked databricks + sdk so it needed to be added here""" + instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" + instance_source: Optional[dict] = None + """An APC attribute only. This field is missing in the API docs and the unforked databricks + sdk so it needed to be added here""" + jdbc_port: Optional[int] = None """Port on which Spark JDBC server is listening, in the driver nod. No service will be listeningon on this port in executor nodes.""" + last_activity_time: Optional[int] = None + """An APC attribute only. This field is missing in the API docs and the unforked databricks + sdk so it needed to be added here""" + last_restarted_time: Optional[int] = None """the timestamp that the cluster was started/restarted""" @@ -3135,6 +3163,7 @@ class EventDetailsCause(Enum): class EventType(Enum): + ADD_NODES_FAILED = 'ADD_NODES_FAILED' AUTOSCALING_STATS_REPORT = 'AUTOSCALING_STATS_REPORT' CREATING = 'CREATING' DBFS_DOWN = 'DBFS_DOWN' diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 27cb330ff..3ff787ea7 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -58,8 +58,8 @@ def from_dict(cls, d: Dict[str, any]) -> BaseJob: class BaseRun: attempt_number: Optional[int] = None """The sequence number of this run attempt for a triggered job run. The initial attempt of a run - has an attempt_number of 0\. If the initial run attempt fails, and the job has a retry policy - (`max_retries` \> 0), subsequent runs are created with an `original_attempt_run_id` of the + has an attempt_number of 0. If the initial run attempt fails, and the job has a retry policy + (`max_retries` > 0), subsequent runs are created with an `original_attempt_run_id` of the original attempt’s ID and an incrementing `attempt_number`. Runs are retried only until they succeed, and the maximum `attempt_number` is the same as the `max_retries` value for the job.""" @@ -2505,8 +2505,8 @@ class Run: attempt_number: Optional[int] = None """The sequence number of this run attempt for a triggered job run. The initial attempt of a run - has an attempt_number of 0\. If the initial run attempt fails, and the job has a retry policy - (`max_retries` \> 0), subsequent runs are created with an `original_attempt_run_id` of the + has an attempt_number of 0. If the initial run attempt fails, and the job has a retry policy + (`max_retries` > 0), subsequent runs are created with an `original_attempt_run_id` of the original attempt’s ID and an incrementing `attempt_number`. Runs are retried only until they succeed, and the maximum `attempt_number` is the same as the `max_retries` value for the job.""" @@ -3344,15 +3344,16 @@ def from_dict(cls, d: Dict[str, any]) -> RunState: class RunTask: """Used when outputting a child run, in GetRun or ListRuns.""" - task_key: str + # adding the default here is necessary to process legacy cluster reports + task_key: str = None """A unique name for the task. This field is used to refer to this task from other tasks. This field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset.""" attempt_number: Optional[int] = None """The sequence number of this run attempt for a triggered job run. The initial attempt of a run - has an attempt_number of 0\. If the initial run attempt fails, and the job has a retry policy - (`max_retries` \> 0), subsequent runs are created with an `original_attempt_run_id` of the + has an attempt_number of 0. If the initial run attempt fails, and the job has a retry policy + (`max_retries` > 0), subsequent runs are created with an `original_attempt_run_id` of the original attempt’s ID and an incrementing `attempt_number`. Runs are retried only until they succeed, and the maximum `attempt_number` is the same as the `max_retries` value for the job.""" From 57a2b067ec435825eaa22a0ba5400cbeadb31381 Mon Sep 17 00:00:00 2001 From: CaymanWilliams Date: Tue, 2 Jul 2024 08:59:08 -0600 Subject: [PATCH 2/3] remove escape fix in docstrings --- databricks/sdk/service/jobs.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 3ff787ea7..46db98a95 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -58,8 +58,8 @@ def from_dict(cls, d: Dict[str, any]) -> BaseJob: class BaseRun: attempt_number: Optional[int] = None """The sequence number of this run attempt for a triggered job run. The initial attempt of a run - has an attempt_number of 0. If the initial run attempt fails, and the job has a retry policy - (`max_retries` > 0), subsequent runs are created with an `original_attempt_run_id` of the + has an attempt_number of 0\. If the initial run attempt fails, and the job has a retry policy + (`max_retries` \> 0), subsequent runs are created with an `original_attempt_run_id` of the original attempt’s ID and an incrementing `attempt_number`. Runs are retried only until they succeed, and the maximum `attempt_number` is the same as the `max_retries` value for the job.""" @@ -2505,8 +2505,8 @@ class Run: attempt_number: Optional[int] = None """The sequence number of this run attempt for a triggered job run. The initial attempt of a run - has an attempt_number of 0. If the initial run attempt fails, and the job has a retry policy - (`max_retries` > 0), subsequent runs are created with an `original_attempt_run_id` of the + has an attempt_number of 0\. If the initial run attempt fails, and the job has a retry policy + (`max_retries` \> 0), subsequent runs are created with an `original_attempt_run_id` of the original attempt’s ID and an incrementing `attempt_number`. Runs are retried only until they succeed, and the maximum `attempt_number` is the same as the `max_retries` value for the job.""" @@ -3352,8 +3352,8 @@ class RunTask: attempt_number: Optional[int] = None """The sequence number of this run attempt for a triggered job run. The initial attempt of a run - has an attempt_number of 0. If the initial run attempt fails, and the job has a retry policy - (`max_retries` > 0), subsequent runs are created with an `original_attempt_run_id` of the + has an attempt_number of 0\. If the initial run attempt fails, and the job has a retry policy + (`max_retries` \> 0), subsequent runs are created with an `original_attempt_run_id` of the original attempt’s ID and an incrementing `attempt_number`. Runs are retried only until they succeed, and the maximum `attempt_number` is the same as the `max_retries` value for the job.""" From 7eedbfbf3e6196067fa345c9a2ddef5e4419598f Mon Sep 17 00:00:00 2001 From: CaymanWilliams Date: Tue, 2 Jul 2024 09:33:38 -0600 Subject: [PATCH 3/3] add jira ticket to comments --- databricks/sdk/service/compute.py | 15 ++++++++------- databricks/sdk/service/jobs.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/databricks/sdk/service/compute.py b/databricks/sdk/service/compute.py index a8b162d25..7f22e57f3 100755 --- a/databricks/sdk/service/compute.py +++ b/databricks/sdk/service/compute.py @@ -787,7 +787,7 @@ class ClusterDetails: - Name: """ disk_spec: Optional[dict] = None - """An APC attribute only. This field is missing in the API docs and the unforked databricks + """[PROD-2198] An APC attribute only. This field is missing in the API docs and the unforked databricks sdk so it needed to be added here""" docker_image: Optional[DockerImage] = None @@ -797,11 +797,11 @@ class ClusterDetails: Databricks application that manages the per-notebook Spark REPLs.""" driver_healthy: Optional[bool] = None - """An APC attribute only. This field is missing in the API docs and the unforked databricks + """[PROD-2198] An APC attribute only. This field is missing in the API docs and the unforked databricks sdk so it needed to be added here""" driver_instance_source: Optional[dict] = None - """An APC attribute only. This field is missing in the API docs and the unforked databricks + """[PROD-2198] An APC attribute only. This field is missing in the API docs and the unforked databricks sdk so it needed to be added here""" driver_instance_pool_id: Optional[str] = None @@ -813,7 +813,7 @@ class ClusterDetails: type will be set as the same value as `node_type_id` defined above.""" effective_spark_version: Optional[str] = None - """An APC attribute only. This field is missing in the API docs and the unforked databricks + """[PROD-2198] An APC attribute only. This field is missing in the API docs and the unforked databricks sdk so it needed to be added here""" enable_elastic_disk: Optional[bool] = None @@ -837,14 +837,14 @@ class ClusterDetails: init script logs are sent to `//init_scripts`.""" init_scripts_safe_mode: Optional[int] = None - """An APC attribute only. This field is missing in the API docs and the unforked databricks + """[PROD-2198] An APC attribute only. This field is missing in the API docs and the unforked databricks sdk so it needed to be added here""" instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" instance_source: Optional[dict] = None - """An APC attribute only. This field is missing in the API docs and the unforked databricks + """[PROD-2198] An APC attribute only. This field is missing in the API docs and the unforked databricks sdk so it needed to be added here""" jdbc_port: Optional[int] = None @@ -852,7 +852,7 @@ class ClusterDetails: on this port in executor nodes.""" last_activity_time: Optional[int] = None - """An APC attribute only. This field is missing in the API docs and the unforked databricks + """[PROD-2198] An APC attribute only. This field is missing in the API docs and the unforked databricks sdk so it needed to be added here""" last_restarted_time: Optional[int] = None @@ -3169,6 +3169,7 @@ class EventDetailsCause(Enum): class EventType(Enum): + # [PROD-2198] Test data in the backend has an event type that was missing here ADD_NODES_FAILED = 'ADD_NODES_FAILED' AUTOSCALING_STATS_REPORT = 'AUTOSCALING_STATS_REPORT' CREATING = 'CREATING' diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 49b679c00..808e785c0 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -3408,7 +3408,7 @@ def from_dict(cls, d: Dict[str, any]) -> RunState: class RunTask: """Used when outputting a child run, in GetRun or ListRuns.""" - # adding the default here is necessary to process legacy cluster reports + # [PROD-2198] adding the default here is necessary to process legacy cluster reports task_key: str = None """A unique name for the task. This field is used to refer to this task from other tasks. This field is required and must be unique within its parent job. On Update or Reset, this field is