To celebrate the release of GATK 4.0, we are giving away free credits for running the GATK4 Best Practices pipelines in FireCloud, our secure online analysis portal. It’s first come first serve, so sign up now to claim your free credits worth $250. Sponsored by Google Cloud. Learn more at https://software.broadinstitute.org/firecloud/documentation/freecredits

Selectively disable call caching?

mmahmmah Member, Broadie

I am having performance issues with call caching in Cromwell v27. I suspect that the problem is related to a task that takes as input the human reference and a bam file, and is scattered over hundreds of bam files. The act of hashing all of the inputs for each task may be taking longer than the tasks themselves would.

Is there a way I can disable call caching for this set of tasks selectively, but not for other tasks?

In case my diagnosis is incorrect, this is an excerpt from the job metadata. I have the complete file as well.

{
      "executionStatus": "Done",
      "stdout": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/stdout",
      "shardIndex": 252,
      "outputs": {
        "snp_target_stats": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/snp_target_stats"
      },
      "runtimeAttributes": {
        "runtime_minutes": "600",
        "queue": "short",
        "requested_memory_mb_per_core": "8000",
        "failOnStderr": "false",
        "continueOnReturnCode": "0",
        "cpus": "2"
      },
      "callCaching": {
        "allowResultReuse": true,
        "effectiveCallCachingMode": "ReadAndWriteCache",
        "hit": true,
        "result": "Cache Hit: 2339bf4d-f580-4d32-9c39-eb845dfb6dca:ancientDNA_screen.spike3k_pre:252"
      },
      "inputs": {
        "reference": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa",
        "coordinates": "spike3000.hs37d5.coordinates",
        "reference_sa": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.sa",
        "python_snp_target": "snp_target_coverage.py",
        "minimum_base_quality": 20,
        "label": "spike3k_pre",
        "excessive_mismatch_penalty": 50,
        "reference_amb": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.amb",
        "reference_fai": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.fai",
        "reference_bwt": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.bwt",
        "minimum_mapping_quality": 37,
        "reference_pac": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.pac",
        "sample_id_filename": "3-91-Q32-Q7.bam",
        "picard_jar": "picard-v2.9.2.jar",
        "bam": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-demultiplex_hs37d5/execution/glob-3bcbe4e7489c90f75e0523ac6f3a9385/3-91-Q32-Q7.bam",
        "reference_ann": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-prepare_reference_hs37d5/execution/hs37d5.fa.ann"
      },
      "returnCode": 0,
      "backend": "LSF",
      "end": "2017-06-12T18:15:03.218-04:00",
      "stderr": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252/execution/stderr",
      "callRoot": "/n/scratch2/mym11/fulltest/cromwell-executions/ancientDNA_screen/28436ca6-a230-42e0-9f56-29b19f738b89/call-spike3k_pre/shard-252",
      "attempt": 1,
      "executionEvents": [{
        "startTime": "2017-06-12T18:14:49.080-04:00",
        "description": "BackendIsCopyingCachedOutputs",
        "endTime": "2017-06-12T18:15:01.731-04:00"
      }, {
        "startTime": "2017-06-12T18:15:03.218-04:00",
        "description": "UpdatingJobStore",
        "endTime": "2017-06-12T18:15:03.218-04:00"
      }, {
        "startTime": "2017-06-12T18:14:49.077-04:00",
        "description": "FetchingCachedOutputsFromDatabase",
        "endTime": "2017-06-12T18:14:49.080-04:00"
      }, {
        "startTime": "2017-06-12T17:13:21.218-04:00",
        "description": "CheckingCallCache",
        "endTime": "2017-06-12T18:14:49.077-04:00"
      }, {
        "startTime": "2017-06-12T13:34:35.058-04:00",
        "description": "Pending",
        "endTime": "2017-06-12T13:34:35.058-04:00"
      }, {
        "startTime": "2017-06-12T17:13:21.217-04:00",
        "description": "PreparingJob",
        "endTime": "2017-06-12T17:13:21.218-04:00"
      }, {
        "startTime": "2017-06-12T13:34:35.058-04:00",
        "description": "RequestingExecutionToken",
        "endTime": "2017-06-12T17:13:21.217-04:00"
      }, {
        "startTime": "2017-06-12T18:15:01.731-04:00",
        "description": "UpdatingCallCache",
        "endTime": "2017-06-12T18:15:03.218-04:00"
      }],
      "start": "2017-06-12T13:34:35.058-04:00"
    }

Best Answer

Answers

Sign In or Register to comment.