On Monday and Tuesday, November 12-13, the communications team will be out of the office for a U.S. federal holiday and a team event. We will be back in action on November 14th and apologize for any inconvenience this may cause. Thank you for using the forum.

Selectively disable call caching?

mmahmmah Member, Broadie

I am having performance issues with call caching in Cromwell v27. I suspect that the problem is related to a task that takes as input the human reference and a bam file, and is scattered over hundreds of bam files. The act of hashing all of the inputs for each task may be taking longer than the tasks themselves would.

Is there a way I can disable call caching for this set of tasks selectively, but not for other tasks?

In case my diagnosis is incorrect, this is an excerpt from the job metadata. I have the complete file as well.

{
      "executionStatus": "Done",
      "stdout": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/stdout",
      "shardIndex": 252,
      "outputs": {
        "snp_target_stats": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/snp_target_stats"
      },
      "runtimeAttributes": {
        "runtime_minutes": "600",
        "queue": "short",
        "requested_memory_mb_per_core": "8000",
        "failOnStderr": "false",
        "continueOnReturnCode": "0",
        "cpus": "2"
      },
      "callCaching": {
        "allowResultReuse": true,
        "effectiveCallCachingMode": "ReadAndWriteCache",
        "hit": true,
        "result": "Cache Hit: 2339bf4d-f580-4d32-9c39-eb845dfb6dca:ancientDNA_screen.spike3k_pre:252"
      },
      "inputs": {
        "reference": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa",
        "coordinates": "spike3000.hs37d5.coordinates",
        "reference_sa": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.sa",
        "python_snp_target": "snp_target_coverage.py",
        "minimum_base_quality": 20,
        "label": "spike3k_pre",
        "excessive_mismatch_penalty": 50,
        "reference_amb": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.amb",
        "reference_fai": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.fai",
        "reference_bwt": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.bwt",
        "minimum_mapping_quality": 37,
        "reference_pac": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.pac",
        "sample_id_filename": "3-91-Q32-Q7.bam",
        "picard_jar": "picard-v2.9.2.jar",
        "bam": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-demultiplex_hs37d5/execution/glob-3bcbe4e7489c90f75e0523ac6f3a9385/3-91-Q32-Q7.bam",
        "reference_ann": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.ann"
      },
      "returnCode": 0,
      "backend": "LSF",
      "end": "2017-06-12T18:15:03.218-04:00",
      "stderr": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/stderr",
      "callRoot": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252",
      "attempt": 1,
      "executionEvents": [{
        "startTime": "2017-06-12T18:14:49.080-04:00",
        "description": "BackendIsCopyingCachedOutputs",
        "endTime": "2017-06-12T18:15:01.731-04:00"
      }, {
        "startTime": "2017-06-12T18:15:03.218-04:00",
        "description": "UpdatingJobStore",
        "endTime": "2017-06-12T18:15:03.218-04:00"
      }, {
        "startTime": "2017-06-12T18:14:49.077-04:00",
        "description": "FetchingCachedOutputsFromDatabase",
        "endTime": "2017-06-12T18:14:49.080-04:00"
      }, {
        "startTime": "2017-06-12T17:13:21.218-04:00",
        "description": "CheckingCallCache",
        "endTime": "2017-06-12T18:14:49.077-04:00"
      }, {
        "startTime": "2017-06-12T13:34:35.058-04:00",
        "description": "Pending",
        "endTime": "2017-06-12T13:34:35.058-04:00"
      }, {
        "startTime": "2017-06-12T17:13:21.217-04:00",
        "description": "PreparingJob",
        "endTime": "2017-06-12T17:13:21.218-04:00"
      }, {
        "startTime": "2017-06-12T13:34:35.058-04:00",
        "description": "RequestingExecutionToken",
        "endTime": "2017-06-12T17:13:21.217-04:00"
      }, {
        "startTime": "2017-06-12T18:15:01.731-04:00",
        "description": "UpdatingCallCache",
        "endTime": "2017-06-12T18:15:03.218-04:00"
      }],
      "start": "2017-06-12T13:34:35.058-04:00"
    }

Best Answer

Answers

Sign In or Register to comment.