<script setup lang="ts"> import BootstrapModal from "@/components/modals/BootstrapModal.vue"; import { type ResourceVersionOut, Status } from "@/client/resource"; import { computed, onMounted, ref, watch } from "vue"; import { environment } from "@/environment"; import CopyToClipboardIcon from "@/components/CopyToClipboardIcon.vue"; import { useS3KeyStore } from "@/stores/s3keys"; import type { S3Key } from "@/client/s3proxy"; import { useS3ObjectStore } from "@/stores/s3objects"; import { useResourceStore } from "@/stores/resources"; import FontAwesomeIcon from "@/components/FontAwesomeIcon.vue"; import { Modal } from "bootstrap"; import RequestReviewButton from "@/components/resources/RequestReviewButton.vue"; const props = defineProps<{ modalId: string; resourceVersion?: ResourceVersionOut; }>(); const s3KeyRepository = useS3KeyStore(); const objectRepository = useS3ObjectStore(); const resourceRepository = useResourceStore(); let infoResourceModal: Modal | null = null; let refreshTimeout: NodeJS.Timeout | undefined = undefined; enum Tool { PYTHON = "python", S5CMD = "s5cmd", MINIO = "minio", } watch( () => props.resourceVersion, (newVersion, oldVersion) => { if (newVersion?.resource_version_id !== oldVersion?.resource_version_id) { resourceReviewEnabled.value = false; if (newVersion?.status === Status.RESOURCE_REQUESTED) { checkS3Resource(newVersion); } } }, ); const activeTool = ref<Tool>(Tool.S5CMD); const resourceReviewEnabled = ref<boolean>(false); const resourceS3Path = computed<string>(() => { return ( props.resourceVersion?.s3_path ?? "s3://examplebucket/RESOURCE-ID/RESOURCE-VERSION-ID/resource.tar.gz" ); }); const resourceMinioS3Path = computed<string>(() => { return resourceS3Path.value.slice(5); }); const resourceBucket = computed<string>(() => { return resourceMinioS3Path.value.split("/")[0]; }); const resourceKey = computed<string>(() => { return resourceS3Path.value.split(resourceBucket.value)[1].slice(1); }); const codeExample = computed<string>(() => { if (activeTool.value === Tool.S5CMD) { return `export AWS_REGION="us-west-1" export AWS_ACCESS_KEY_ID="${s3Key.value.access_key}" export AWS_SECRET_ACCESS_KEY="${s3Key.value.secret_key}" export S3_ENDPOINT_URL="${environment.S3_URL}" s5cmd cp --show-progress /PATH/TO/RESOURCE \\ ${resourceS3Path.value}`; } else if (activeTool.value === Tool.MINIO) { return `mc alias set clowm-s3 ${environment.S3_URL} "${s3Key.value.access_key}" "${s3Key.value.secret_key}" mc cp /PATH/TO/RESOURCE \\ clowm-s3/${resourceMinioS3Path.value}`; } else if (activeTool.value === Tool.PYTHON) { return `import boto3 s3 = boto3.resource( service_name="s3", aws_access_key_id="${s3Key.value.access_key}", aws_secret_access_key="${s3Key.value.secret_key}", endpoint_url="${environment.S3_URL}", verify=True, ) with open("/PATH/TO/RESOURCE", "rb") as f: s3.Object( bucket_name="${resourceBucket.value}", key="${resourceKey.value}" ).upload_fileobj(f)`; } return ""; }); const s3Key = computed<S3Key>(() => { return ( s3KeyRepository.keys[0] ?? { access_key: "abc", secret_key: "def", } ); }); function checkS3Resource(resourceVersion: ResourceVersionOut) { const bucket = resourceVersion.s3_path.slice(5).split("/")[0]; const key = resourceVersion.s3_path.split(bucket)[1].slice(1); objectRepository .fetchS3ObjectMeta(bucket, key) .then(() => { resourceReviewEnabled.value = true; }) .catch(() => { resourceReviewEnabled.value = false; }); } function clickCheckS3Resource(resourceVersion: ResourceVersionOut) { clearTimeout(refreshTimeout); refreshTimeout = setTimeout(() => { checkS3Resource(resourceVersion); }, 500); } function requestReview(resourceVersion: ResourceVersionOut) { resourceRepository.requestReview(resourceVersion).then(() => { infoResourceModal?.hide(); }); } onMounted(() => { infoResourceModal = new Modal("#" + props.modalId); }); </script> <template> <bootstrap-modal :modalId="props.modalId" modal-label="Upload Resource Info Modal" sizeModifier="lg" :track-modal-value="resourceVersion?.resource_version_id" > <template #header>How to upload a resource to the cluster</template> <template #body> <ol> <li :class="{ 'text-decoration-line-through': props.resourceVersion }"> <h6> Prerequisite: Prepare a single compressed tar archive of your resource data </h6> <p :hidden="props.resourceVersion != undefined"> The data of your resource must be compressed into a single tar archive to save bandwidth and storage space.<br /> <code>tar -czf resource.tar.gz /PATH/TO/MY/RESOURCE/DATA</code> </p> </li> <li :class="{ 'text-decoration-line-through': props.resourceVersion }"> <h6>Request a resource in CloWM</h6> <p :hidden="props.resourceVersion != undefined"> Click on <b>Create Resource</b> and fill out the form </p> </li> <li class="pb-2" :class="{ 'text-decoration-line-through': resourceReviewEnabled, }" > <h6>Upload the resource</h6> <template v-if="!resourceReviewEnabled"> <p> Upload the tar archive to the provided S3 path with a tool of your choice. </p> <ul class="nav nav-tabs mb-2"> <li class="nav-item"> <a class="nav-link" :class="{ active: activeTool === Tool.S5CMD }" aria-current="page" href="#" @click="activeTool = Tool.S5CMD" >s5cmd</a > </li> <li class="nav-item"> <a class="nav-link" :class="{ active: activeTool === Tool.MINIO }" href="#" @click="activeTool = Tool.MINIO" >Minio</a > </li> <li class="nav-item"> <a class="nav-link" :class="{ active: activeTool === Tool.PYTHON }" href="#" @click="activeTool = Tool.PYTHON" >Python</a > </li> </ul> <template v-if="activeTool === Tool.S5CMD"> <p> <code>s5cmd</code> is a very fast S3 and local filesystem execution tool. It comes with support for a multitude of operations including tab completion and wildcard support for files, which can be very handy for your object storage workflow while working with large number of files. </p> <p> The GitHub repository contains a <a href="https://github.com/peak/s5cmd?tab=readme-ov-file#installation" target="_blank" >installation guide</a >. </p> <pre class="w-100"><code class="hljs language-bash"><span class="hljs-built_in">export</span> AWS_REGION=<span class="hljs-string">"us-west-1"</span> <span class="hljs-built_in">export</span> AWS_ACCESS_KEY_ID=<span class="hljs-string">"{{ s3Key.access_key }}"</span> <span class="hljs-built_in">export</span> AWS_SECRET_ACCESS_KEY=<span class="hljs-string">"{{ s3Key.secret_key }}"</span> <span class="hljs-built_in">export</span> S3_ENDPOINT_URL=<span class="hljs-string">"{{ environment.S3_URL }}"</span> <span class="hljs-built_in">s5cmd</span> cp --show-progress /PATH/TO/RESOURCE \ {{ resourceS3Path }}</code></pre> </template> <template v-else-if="activeTool === Tool.MINIO"> <p> The MinIO Client <code>mc</code> command line tool provides a modern alternative to UNIX commands like <code>ls</code>, <code>cat</code>, <code>cp</code>, <code>mirror</code>, and <code>diff</code> with support for both filesystems and Amazon S3-compatible cloud storage services. </p> <p> The official documentation contains a <a href="https://min.io/docs/minio/linux/reference/minio-mc.html#install-mc" target="_blank" >installation guide</a >. </p> <pre class="w-100" ><code class="hljs language-bash">mc <span class="hljs-built_in">alias</span> <span class="hljs-built_in">set</span> clowm-s3 {{ environment.S3_URL }} <span class="hljs-string">"{{ s3Key.access_key }}"</span> <span class="hljs-string">"{{ s3Key.secret_key }}"</span> mc <span class="hljs-built_in">cp</span> /PATH/TO/RESOURCE \ clowm-s3/{{ resourceMinioS3Path }}</code></pre> </template> <template v-else-if="activeTool === Tool.PYTHON"> <p> You use the AWS SDK for Python (<a href="https://boto3.amazonaws.com/v1/documentation/api/latest/index.html" target="_blank" >Boto3</a >) to create, configure, and manage AWS services, such as Amazon Elastic Compute Cloud (Amazon EC2) and Amazon Simple Storage Service (Amazon S3). The SDK provides an object-oriented API as well as low-level access to AWS services. </p> <p> The package can be installed with pip: <code>pip install boto3</code> </p> <pre class="w-100" ><code class="hljs language-python"><span class="hljs-keyword">import</span> boto3 s3 = boto3.resource( service_name=<span class="hljs-string">"s3"</span>, aws_access_key_id=<span class="hljs-string">"{{ s3Key.access_key }}"</span>, aws_secret_access_key=<span class="hljs-string">"{{ s3Key.secret_key }}"</span>, endpoint_url=<span class="hljs-string">"{{ environment.S3_URL }}"</span>, verify=<span class="hljs-literal">True</span>, ) <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"/PATH/TO/RESOURCE"</span>, <span class="hljs-string">"rb"</span>) <span class="hljs-keyword">as</span> f: s3.Object( bucket_name=<span class="hljs-string">"{{ resourceBucket }}"</span>, key=<span class="hljs-string">"{{ resourceKey }}"</span> ).upload_fileobj(f)</code></pre> </template> <b v-if="resourceVersion == undefined" class="text-danger"> <font-awesome-icon icon="fa-solid fa-triangle-exclamation" class="me-1" /> Important: This is just example code <font-awesome-icon icon="fa-solid fa-triangle-exclamation" class="ms-1" /> </b> </template> <copy-to-clipboard-icon v-if="props.resourceVersion && !resourceReviewEnabled" button :text="codeExample" /> </li> <li> <h6>Request review</h6> <p> Click <b>Request Review</b> to request a review of the resource. </p> <request-review-button v-if="props.resourceVersion?.status === Status.RESOURCE_REQUESTED" class="mb-2" :disabled="!resourceReviewEnabled" @click-review="requestReview(props.resourceVersion)" @click-refresh="clickCheckS3Resource(props.resourceVersion)" /> </li> <li> <h6>Request resource synchronization</h6> <p> Once a Reviewer approves your resource, the resource will be publicly visible in CloWM. To use it in during a workflow execution, you can request the synchronization of the resource to the cluster. </p> </li> <li> <h6>Resource availability</h6> <p> When an administrator approves the synchronization request and the resource is download to the cluster, the resource will now available for a workflow execution and is accessible for every workflow via its <b>Nextflow Access Path</b>. </p> </li> </ol> </template> <template #footer> <button type="button" class="btn btn-secondary" data-bs-dismiss="modal"> Close </button> </template> </bootstrap-modal> </template> <style scoped></style>