Update: July 26, 2019
This section of the forum is now closed; we are working on a new support model for WDL that we will share here shortly. For Cromwell-specific issues, see the Cromwell docs and post questions on Github.

Selectively disable call caching?

mmahmmah ✭✭Member, Broadie ✭✭

I am having performance issues with call caching in Cromwell v27. I suspect that the problem is related to a task that takes as input the human reference and a bam file, and is scattered over hundreds of bam files. The act of hashing all of the inputs for each task may be taking longer than the tasks themselves would.

Is there a way I can disable call caching for this set of tasks selectively, but not for other tasks?

In case my diagnosis is incorrect, this is an excerpt from the job metadata. I have the complete file as well.

{
      "executionStatus": "Done",
      "stdout": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/stdout",
      "shardIndex": 252,
      "outputs": {
        "snp_target_stats": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/snp_target_stats"
      },
      "runtimeAttributes": {
        "runtime_minutes": "600",
        "queue": "short",
        "requested_memory_mb_per_core": "8000",
        "failOnStderr": "false",
        "continueOnReturnCode": "0",
        "cpus": "2"
      },
      "callCaching": {
        "allowResultReuse": true,
        "effectiveCallCachingMode": "ReadAndWriteCache",
        "hit": true,
        "result": "Cache Hit: 2339bf4d-f580-4d32-9c39-eb845dfb6dca:ancientDNA_screen.spike3k_pre:252"
      },
      "inputs": {
        "reference": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa",
        "coordinates": "spike3000.hs37d5.coordinates",
        "reference_sa": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.sa",
        "python_snp_target": "snp_target_coverage.py",
        "minimum_base_quality": 20,
        "label": "spike3k_pre",
        "excessive_mismatch_penalty": 50,
        "reference_amb": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.amb",
        "reference_fai": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.fai",
        "reference_bwt": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.bwt",
        "minimum_mapping_quality": 37,
        "reference_pac": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.pac",
        "sample_id_filename": "3-91-Q32-Q7.bam",
        "picard_jar": "picard-v2.9.2.jar",
        "bam": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-demultiplex_hs37d5/execution/glob-3bcbe4e7489c90f75e0523ac6f3a9385/3-91-Q32-Q7.bam",
        "reference_ann": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.ann"
      },
      "returnCode": 0,
      "backend": "LSF",
      "end": "2017-06-12T18:15:03.218-04:00",
      "stderr": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/stderr",
      "callRoot": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252",
      "attempt": 1,
      "executionEvents": [{
        "startTime": "2017-06-12T18:14:49.080-04:00",
        "description": "BackendIsCopyingCachedOutputs",
        "endTime": "2017-06-12T18:15:01.731-04:00"
      }, {
        "startTime": "2017-06-12T18:15:03.218-04:00",
        "description": "UpdatingJobStore",
        "endTime": "2017-06-12T18:15:03.218-04:00"
      }, {
        "startTime": "2017-06-12T18:14:49.077-04:00",
        "description": "FetchingCachedOutputsFromDatabase",
        "endTime": "2017-06-12T18:14:49.080-04:00"
      }, {
        "startTime": "2017-06-12T17:13:21.218-04:00",
        "description": "CheckingCallCache",
        "endTime": "2017-06-12T18:14:49.077-04:00"
      }, {
        "startTime": "2017-06-12T13:34:35.058-04:00",
        "description": "Pending",
        "endTime": "2017-06-12T13:34:35.058-04:00"
      }, {
        "startTime": "2017-06-12T17:13:21.217-04:00",
        "description": "PreparingJob",
        "endTime": "2017-06-12T17:13:21.218-04:00"
      }, {
        "startTime": "2017-06-12T13:34:35.058-04:00",
        "description": "RequestingExecutionToken",
        "endTime": "2017-06-12T17:13:21.217-04:00"
      }, {
        "startTime": "2017-06-12T18:15:01.731-04:00",
        "description": "UpdatingCallCache",
        "endTime": "2017-06-12T18:15:03.218-04:00"
      }],
      "start": "2017-06-12T13:34:35.058-04:00"
    }

Best Answer

Answers

Sign In or Register to comment.