Unable to run workflow from command-line

classic Classic list List threaded Threaded
1 message Options
| Threaded
Open this post in threaded view
|

Unable to run workflow from command-line

Aarthi Mohan
Hi all,

Aplogies for the long message!

I am trying to execute a galaxy workflow from command-line using bioblend. The workflow invloves bwa, gatk, samtools mpileup, bcftools and works with dataset paired-end collections. The following is the script used to achieve it and I am unable to get the workflow to run


from bioblend.galaxy import GalaxyInstance
from bioblend.galaxy.histories import HistoryClient
from bioblend.galaxy.tools import ToolClient
from bioblend.galaxy.workflows import WorkflowClient
from bioblend.galaxy.datasets import DatasetClient
from bioblend.galaxy import dataset_collections as collections
import os,re

url , key  = "http://#.#.#.#:9090", "c5fe47950c99c52a95c6de2542599ddb"
galaxyInstance = GalaxyInstance(url=url, key=key)
historyClient = HistoryClient(galaxyInstance)
toolClient = ToolClient(galaxyInstance)
workflowClient = WorkflowClient(galaxyInstance)
datasetClient = DatasetClient(galaxyInstance)
workflow_list = workflowClient.get_workflows()
 
bwa_workflow_id = [ i['id']  for i in workflow_list if i['name'].startswith('bwa') ][0] #desired workflow id
bwa_workflow_details = workflowClient.show_workflow(bwa_workflow_id) #workflow details like input, steps, etc.,
bwa_workflow_input_id = bwa_workflow_details['inputs'].keys()[0] #workflow input ids
created_history = historyClient.create_history(name='testing_bwa2bctools2')
print 'created a new history with name ' , created_history['name']
path = "/home/sphadmin/data/fastq_files/"
files = {i:os.path.join(path,i) for i in os.listdir(path) if os.path.splitext(i)[1] == '.fastq'}
hda = {i:toolClient.upload_file(files[i], created_history['id'], type='fastq',dbkey='H37Rv') for i in files}
files_dataset_id = {i:hda[i]['outputs'][0]['id'] for i in files}
print 'uploading fastq files to history'

fwd = re.compile(r'R1')
paired_list = {}
for i in files:
if fwd.search(i):
prefix, file = i.split('_R1')[0] , i
if not prefix in paired_list:
paired_list[prefix] = {}
paired_list[prefix]['forward'] = file
else:
prefix, file = i.split('_R2')[0], i
if not prefix  in paired_list:
paired_list[prefix] = {}
paired_list[prefix]['reverse'] = file

paired_collection = [collections.CollectionElement(name=i,type='paired',elements=[collections.HistoryDatasetElement(name=j,id=files_dataset_id[paired_list[i][j]]) for j in paired_list[i] ]) for i in paired_list]
collectionDesc = collections.CollectionDescription(name='My collection list', type='list:paired', elements=paired_collection)
dataset_collection = historyClient.create_dataset_collection(history_id=created_history['id'], collection_description=collectionDesc)
print 'creating dataset collection in history'
data_wstep_id = {bwa_workflow_input_id: {'id': dataset_collection['id'], 'src':'hda'}}
data_wstep_index = {"0": {'id': dataset_collection['id'], 'src':'hda'}}
exec_wf = workflowClient.invoke_workflow(bwa_workflow_id, inputs=data_wstep_index,history_id=created_history['id'])
print 'submitting workflow'



I have tried using the step_id with run_workflow() and get the following error, 

workflowClient.run_workflow(bwa_workflow_id, data_wstep_id, history_id=created_history['id'])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/bioblend/galaxy/workflows/__init__.py", line 271, in run_workflow
    return Client._post(self, payload)
  File "/usr/local/lib/python2.7/dist-packages/bioblend/galaxy/client.py", line 157, in _post
    files_attached=files_attached)
  File "/usr/local/lib/python2.7/dist-packages/bioblend/galaxyclient.py", line 134, in make_post_request
    body=r.text, status_code=r.status_code)
bioblend.ConnectionError: Unexpected HTTP status code: 500: {"err_msg": "Uncaught exception in exposed API method:", "err_code": 0}

step_order_index with invoke_workflow() does not produce any error, but workflow is executed. So 'exec_wf' returns with failed 'status'. 

{u'inputs': {}, u'update_time': u'2017-05-11T09:13:31', u'uuid': u'199adb04-362a-11e7-90e9-1cc1de6d5ef4', u'history_id': u'fd767e8b7355e21a', u'state': u'failed', u'workflow_id': u'1fad1eaf5f4f1766', u'steps': [{u'update_time': u'2017-05-11T09:13:31', u'job_id': None, u'state': None, u'order_index': 0, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'a73186a6d4577bbf', u'id': u'd882ad2044776478'}], u'model_class': u'WorkflowInvocation', u'id': u'964b37715ec9bd22'}

However, If I run the workflow from UI, everything goes fine. Upon checking the invocation details of the workflow run from UI, the 'inputs' is still empty (there is no associated dataset collection id)

{u'inputs': {}, u'update_time': u'2017-05-09T03:44:31', u'uuid': u'c495fef0-3466-11e7-90e9-1cc1de6d5ef4', u'history_id': u'f158df71bb77bb86', u'state': u'scheduled', u'workflow_id': u'1fad1eaf5f4f1766', u'steps': [{u'update_time': u'2017-05-09T03:22:47', u'job_id': u'48885bbd1215db53', u'state': u'ok', u'order_index': 1, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'4df0bfb0de34905a', u'id': u'50807609e10e339d'}, {u'update_time': u'2017-05-09T03:22:47', u'job_id': None, u'state': None, u'order_index': 0, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'a73186a6d4577bbf', u'id': u'f5e1495a2ebc6080'}, {u'update_time': u'2017-05-09T03:22:47', u'job_id': u'f1cb3a6ff6a93c01', u'state': u'ok', u'order_index': 1, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'4df0bfb0de34905a', u'id': u'1d972c3873de3f79'}, {u'update_time': u'2017-05-09T03:22:50', u'job_id': u'822aeedf25d30c76', u'state': u'ok', u'order_index': 2, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'89e52cb393afc192', u'id': u'89788b3f6c8e75f4'}, {u'update_time': u'2017-05-09T03:22:50', u'job_id': u'4fb563a6a1060ab1', u'state': u'ok', u'order_index': 2, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'89e52cb393afc192', u'id': u'b4666ccc5542d171'}, {u'update_time': u'2017-05-09T03:22:53', u'job_id': u'cab68a1ccca17e68', u'state': u'ok', u'order_index': 3, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'b6b7934af4ebec63', u'id': u'3dfe330b26f78ced'}, {u'update_time': u'2017-05-09T03:22:53', u'job_id': u'bb80841fd9d212dd', u'state': u'ok', u'order_index': 3, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'b6b7934af4ebec63', u'id': u'f89ec36d748e8f90'}, {u'update_time': u'2017-05-09T03:22:57', u'job_id': u'02b1cfb59f716d61', u'state': u'ok', u'order_index': 4, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'dba3bb6dfae451b8', u'id': u'dbc267d11443f9a4'}, {u'update_time': u'2017-05-09T03:22:57', u'job_id': u'705cc5c1fdaa7d3e', u'state': u'ok', u'order_index': 4, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'dba3bb6dfae451b8', u'id': u'67ce804af6ec796b'}, {u'update_time': u'2017-05-09T03:22:59', u'job_id': u'665a42f7453c0065', u'state': u'ok', u'order_index': 5, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'5935f39e477d88b2', u'id': u'09d8568e61ebd1d9'}, {u'update_time': u'2017-05-09T03:22:59', u'job_id': u'4488b53dbfdabfab', u'state': u'ok', u'order_index': 5, u'action': None, u'model_class': u'WorkflowInvocationStep', u'workflow_step_id': u'5935f39e477d88b2', u'id': u'b3e4e0c76fe49b64'}], u'model_class': u'WorkflowInvocation', u'id': u'e89067bb68bee7a0'}

Galaxy Version 15.03 & bioblend Version 0.8.0. 

Appreciate your inputs. 

Thanks,
Aarthi

___________________________________________________________
Please keep all replies on the list by using "reply all"
in your mail client.  To manage your subscriptions to this
and other Galaxy lists, please use the interface at:
  https://lists.galaxyproject.org/

To search Galaxy mailing lists use the unified search at:
  http://galaxyproject.org/search/