Skip to content

use submission api v5 to download submission #90

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

use submission api v5 to download submission #90

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a9322fe
use submission api v5 to download submission
deedee Mar 4, 2020
81a2e9b
change single import
deedee Mar 21, 2020
e2e0029
some fixes on ActionsHelper.java
deedee Mar 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions conf/OnlineReview.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5311,6 +5311,9 @@
<Value>@aws_s3_bucket_dmz@</Value>
</Property>
</Property>
<Property name="submission_api">
<Value>@submission_api_endpoint@</Value>
</Property>
</Config>

<!--
Expand Down
2 changes: 1 addition & 1 deletion lib
Open in desktop
Submodule lib updated from 18b3f1 to e81bce
80 changes: 31 additions & 49 deletions src/java/main/com/cronos/onlinereview/actions/projectdetails/BaseProjectDetailsAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,6 @@
*/
package com.cronos.onlinereview.actions.projectdetails;

import java.io.BufferedReader;
import java.io.Closeable;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import com.amazonaws.services.s3.AmazonS3URI;

import com.cronos.onlinereview.Constants;
import com.cronos.onlinereview.actions.DynamicModelDrivenAction;
import com.cronos.onlinereview.actions.event.EventBusServiceClient;
Expand All @@ -36,11 +15,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.topcoder.management.deliverable.Submission;
import com.topcoder.management.deliverable.SubmissionStatus;
import com.topcoder.management.deliverable.Upload;
import com.topcoder.management.deliverable.UploadManager;
import com.topcoder.management.deliverable.UploadStatus;
import com.topcoder.management.deliverable.*;
import com.topcoder.management.deliverable.persistence.UploadPersistenceException;
import com.topcoder.management.deliverable.search.SubmissionFilterBuilder;
import com.topcoder.management.project.Project;
Expand All @@ -57,23 +32,28 @@
import com.topcoder.search.builder.filter.AndFilter;
import com.topcoder.search.builder.filter.EqualToFilter;
import com.topcoder.search.builder.filter.Filter;
import com.topcoder.servlet.request.ConfigurationException;
import com.topcoder.servlet.request.DisallowedDirectoryException;
import com.topcoder.servlet.request.FileDoesNotExistException;
import com.topcoder.servlet.request.FileUpload;
import com.topcoder.servlet.request.FileUploadResult;
import com.topcoder.servlet.request.PersistenceException;
import com.topcoder.servlet.request.UploadedFile;
import com.topcoder.servlet.request.*;
import com.topcoder.util.config.ConfigManagerException;
import com.topcoder.util.errorhandling.BaseException;
import com.topcoder.util.file.DocumentGenerator;
import com.topcoder.util.file.Template;
import com.topcoder.util.file.fieldconfig.Condition;
import com.topcoder.util.file.fieldconfig.Field;
import com.topcoder.util.file.fieldconfig.Node;
import com.topcoder.util.file.fieldconfig.TemplateFields;
import com.topcoder.util.file.templatesource.FileTemplateSource;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* This is the base class for project details actions classes. It provides the
* basic functions which will be used by all project detail actions.
Expand Down Expand Up @@ -200,7 +180,7 @@ protected CorrectnessCheckResult checkForCorrectUploadId(HttpServletRequest requ
protected void processSubmissionDownload(Upload upload, HttpServletRequest request, HttpServletResponse response)
throws UploadPersistenceException, SearchBuilderException, DisallowedDirectoryException,
ConfigurationException, PersistenceException, FileDoesNotExistException, IOException,
ResourcePersistenceException {
ResourcePersistenceException, BaseException {

// At this point, redirect-after-login attribute should be removed (if it
// exists)
Expand Down Expand Up @@ -240,20 +220,22 @@ protected void processSubmissionDownload(Upload upload, HttpServletRequest reque

outputDownloadedFile(uploadedFile, contentDisposition, response);
} else {
AmazonS3URI s3Uri = ActionsHelper.isS3Url(upload.getUrl());
if (s3Uri == null) {
response.sendRedirect(upload.getUrl());
} else {
String key = s3Uri.getKey();
String contentDisposition;
if (submission != null) {
contentDisposition = "attachment; filename=\"submission-" + submission.getId() + "-" + key + "\"";
} else {
contentDisposition = "attachment; filename=\"upload-" + upload.getId() + "-" + key + "\"";
}

ActionsHelper.outputDownloadS3File(upload.getUrl(), key, contentDisposition, response);
}
// AmazonS3URI s3Uri = ActionsHelper.isS3Url(upload.getUrl());
// if (s3Uri == null) {
// response.sendRedirect(upload.getUrl());
// } else {
// String key = s3Uri.getKey();
// String contentDisposition;
// if (submission != null) {
// contentDisposition = "attachment; filename=\"submission-" + submission.getId() + "-" + key + "\"";
// } else {
// contentDisposition = "attachment; filename=\"upload-" + upload.getId() + "-" + key + "\"";
// }
//
// ActionsHelper.outputDownloadS3File(upload.getUrl(), key, contentDisposition, response);
// }
// HttpResponse submissionFromApi = ActionsHelper.downloadSubmissionFromApi(submission.getId());
ActionsHelper.outputDownloadFromSubmissionApi(submission.getId(), response);
}
}

Expand Down
174 changes: 123 additions & 51 deletions src/java/main/com/cronos/onlinereview/util/ActionsHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,46 +3,11 @@
*/
package com.cronos.onlinereview.util;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.math.MathContext;
import java.net.URL;
import java.rmi.RemoteException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.text.Format;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.AmazonS3URI;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.AmazonS3URI;

import javax.ejb.CreateException;
import javax.naming.Context;
import javax.naming.NamingException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import com.cronos.onlinereview.Constants;
import com.cronos.onlinereview.dataaccess.ProjectDataAccess;
import com.cronos.onlinereview.dataaccess.ResourceDataAccess;
Expand All @@ -58,21 +23,15 @@
import com.cronos.termsofuse.dao.ProjectTermsOfUseDao;
import com.cronos.termsofuse.dao.TermsOfUseDao;
import com.cronos.termsofuse.dao.UserTermsOfUseDao;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.opensymphony.xwork2.ActionContext;
import com.opensymphony.xwork2.ActionSupport;
import com.opensymphony.xwork2.TextProvider;
import com.topcoder.date.workdays.DefaultWorkdaysFactory;
import com.topcoder.date.workdays.Workdays;
import com.topcoder.db.connectionfactory.ConfigurationException;
import com.topcoder.db.connectionfactory.DBConnectionException;
import com.topcoder.db.connectionfactory.DBConnectionFactory;
import com.topcoder.db.connectionfactory.DBConnectionFactoryImpl;
import com.topcoder.db.connectionfactory.UnknownConnectionException;
import com.topcoder.management.deliverable.Deliverable;
import com.topcoder.management.deliverable.DeliverableManager;
import com.topcoder.management.deliverable.Submission;
import com.topcoder.management.deliverable.Upload;
import com.topcoder.management.deliverable.UploadManager;
import com.topcoder.db.connectionfactory.*;
import com.topcoder.management.deliverable.*;
import com.topcoder.management.deliverable.late.LateDeliverable;
import com.topcoder.management.deliverable.late.LateDeliverableManager;
import com.topcoder.management.deliverable.persistence.DeliverableCheckingException;
Expand Down Expand Up @@ -113,11 +72,7 @@
import com.topcoder.project.phases.template.StartDateGenerator;
import com.topcoder.project.phases.template.persistence.XmlPhaseTemplatePersistence;
import com.topcoder.search.builder.SearchBuilderException;
import com.topcoder.search.builder.filter.AndFilter;
import com.topcoder.search.builder.filter.EqualToFilter;
import com.topcoder.search.builder.filter.Filter;
import com.topcoder.search.builder.filter.NotFilter;
import com.topcoder.search.builder.filter.OrFilter;
import com.topcoder.search.builder.filter.*;
import com.topcoder.servlet.request.DisallowedDirectoryException;
import com.topcoder.servlet.request.FileUpload;
import com.topcoder.servlet.request.LocalFileUpload;
Expand All @@ -131,6 +86,33 @@
import com.topcoder.web.common.throttle.Throttle;
import com.topcoder.web.ejb.forums.Forums;
import com.topcoder.web.ejb.forums.ForumsHome;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;

import javax.ejb.CreateException;
import javax.naming.Context;
import javax.naming.NamingException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.math.MathContext;
import java.net.URI;
import java.net.URL;
import java.rmi.RemoteException;
import java.sql.*;
import java.text.Format;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.*;


/**
* <p>
Expand Down Expand Up @@ -242,6 +224,23 @@ public class ActionsHelper {
*/
private static final String s3BucketDmz;

/**
* Submission Api endpoint
*/
private static final String submissionApiUrl;

/**
* Submission api get submission
*/
private static final String GET_SUBMISSION_ENDPOINT = "%s/submissions?legacySubmissionId=%s";

/**
* Submission Api download submission
*/
private static final String GET_DOWNLOAD_ENDPOINT = "%s/submissions/%s/download";

protected static final ObjectMapper objectMapper = new ObjectMapper();

static {
try {
ClassLoader loader = ActionsHelper.class.getClassLoader();
Expand All @@ -250,11 +249,14 @@ public class ActionsHelper {
s3BucketDmz = ConfigHelper.getS3BucketDmz();
presignedExpireMillis = ConfigHelper.getPreSignedExpTimeMilis();
s3Client = new AmazonS3Client(new PropertiesCredentials(new File(credentialURL.getFile())));
String sUrl = ConfigHelper.getSubmissionApiEndpoint().trim();
submissionApiUrl = sUrl.endsWith("/") ? sUrl.substring(0, sUrl.length() -1 ) : sUrl;
} catch (Throwable e) {
throw new RuntimeException("Failed load to Amazon S3 CLient", e);
}
}


/**
* This constructor is declared private to prohibit instantiation of the
* <code>ActionsHelper</code> class.
Expand Down Expand Up @@ -3878,4 +3880,74 @@ public static String createStudioLocalFilePath(long projectId, long userId, Stri
buf.append(parameter);
return buf.toString();
}

/**
* Get submission from external API
*
* @param submissionId submissionId
* @throws Exception
*/
public static void outputDownloadFromSubmissionApi(long submissionId,
HttpServletResponse servletResponse) throws BaseException {
HttpResponse response = null;
DefaultHttpClient httpClient = null;
try {
httpClient = new DefaultHttpClient();
log.log(Level.INFO, "Get submission from api id: " + submissionId);
HttpGet request = new HttpGet(String.format(GET_SUBMISSION_ENDPOINT, submissionApiUrl, submissionId));
String v3Token = new JwtTokenUpdater().check().getToken();

request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + v3Token);
request.addHeader(HttpHeaders.ACCEPT, "application/json");
response = httpClient.execute(request);
HttpEntity entity = response.getEntity();

if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
throw new BaseException("Unable submission from the API:" + response.getStatusLine().getReasonPhrase());
}
JsonNode result = objectMapper.readTree(entity.getContent());

if (result.size() < 1) {
throw new BaseException("Can not found submission from API");
}
String v5SubmissionId = result.get(0).get("id").asText();
log.log(Level.INFO, "Found submission id: " + v5SubmissionId);
request.setURI(new URI(String.format(GET_DOWNLOAD_ENDPOINT, submissionApiUrl, v5SubmissionId)));
response = httpClient.execute(request);
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
throw new BaseException("Unable to download submission from the API:" + response.getStatusLine().getReasonPhrase());
}
log.log(Level.INFO, "begin downloading submission: " + v5SubmissionId);
InputStream in = response.getEntity().getContent();

servletResponse.setHeader("Content-Type", response.getEntity().getContentType().getValue());
servletResponse.setStatus(HttpServletResponse.SC_OK);
servletResponse.setIntHeader("Content-Length", (int) response.getEntity().getContentLength());
servletResponse.setHeader("Content-Disposition", response.getHeaders("Content-Disposition")[0].getValue());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@deedee what if there is no Content-Disposition, then there will be index out of boundary issue.


servletResponse.flushBuffer();

OutputStream out = null;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@deedee this is some logic to pipe input stream to output stream, I think it is better code it as a separate method for clearness.

try {
out = servletResponse.getOutputStream();
byte[] buffer = new byte[65536];

for (;;) {
int numOfBytesRead = in.read(buffer);
if (numOfBytesRead == -1) {
break;
}
out.write(buffer, 0, numOfBytesRead);
}
} finally {
in.close();
if (out != null) {
out.close();
}
}

} catch(Exception e) {
throw new BaseException(e.getMessage());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@deedee should include the root cause.

}
}
}
Loading