[HWORKS-681] Flink job apis for hopsworks python library #160
Conversation
| self._project_name = project_name | ||
| self._job_api = job_api.JobsApi(project_id, project_name) | ||
|
|
||
| def get_configuration(self): |
| def get_configuration(self): | ||
| return self._job_api.get_configuration("FLINK") | ||
|
|
||
| def setup_cluster(self, name, config=None): |
| return self._job_api.get_configuration("FLINK") | ||
|
|
||
| def setup_cluster(self, name, config=None): | ||
| if self._job_api.exists(name): |
There was a problem hiding this comment.
This should also validate that the retrieved job is a FLINK job
| self._project_name, | ||
| ) | ||
|
|
||
| def get_jobs(self, execution): |
python/hopsworks/flink_cluster.py
Outdated
| polling_time = 0 | ||
| while polling_time < start_time_out: | ||
| execution = self._execution_api._get(self, execution.id) | ||
| if execution.state == "RUNNING": |
There was a problem hiding this comment.
We should have an else statement here how handle if the job failed while starting up. Otherwise the user will wait 10 minutes and then get the exception thrown on line 43.
Co-authored-by: Robin Andersson <[email protected]>
Co-authored-by: Robin Andersson <[email protected]>
| ) | ||
| return response["files"] | ||
|
|
||
| def upload_jar(self, execution, jar_file): |
There was a problem hiding this comment.
Since we can now do FlinkExecution.upload_jar, can we make this and any other function being called from FlinkCluster/FlinkExecution private?
Co-authored-by: Robin Andersson <[email protected]>
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
There was a problem hiding this comment.
I don't think this snippet works, flink_job is not defined
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
There was a problem hiding this comment.
Same problem, flink_job not defined
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
python/hopsworks/flink_execution.py
Outdated
| flink_cluster = flink_cluster_api.get_cluster(name="myFlinkCluster") | ||
|
|
||
| # get all executions(This will return empty list of no execution is running on this Flink cluster) | ||
| executions = flink_job.get_executions() |
python/hopsworks/flink_execution.py
Outdated
|
|
||
| return self._flink_cluster_api._get_job(self, job_id) | ||
|
|
||
| def stop_execution(self): |
There was a problem hiding this comment.
This function does the same thing as stop()? I think we should just keep the stop() function and remove the stop_execution() one
Co-authored-by: Robin Andersson <[email protected]>
Co-authored-by: Robin Andersson <[email protected]>
There was a problem hiding this comment.
We need to update this file to include the documentation for Flink:
https://github.com/logicalclocks/hopsworks-api/blob/main/auto_doc.py
This comment was marked as outdated.
This comment was marked as outdated.
Sorry, something went wrong.
There was a problem hiding this comment.
You need change the mkdocs.yml file to include the documentation. For example for jobs we do it here
Line 24 in 7b3d6bb
No description provided.