Skip to content

Instantly share code, notes, and snippets.

@ankurcha

ankurcha/Readme.md

Created July 19, 2013 21:23
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ankurcha/6042458 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ankurcha/6042458 to your computer and use it in GitHub Desktop.
Download ZIP
Implementation of the Google Youtube Data API interface * All Channels have a one to one mapping with the entry in the credentialStore. * The refreshing process is managed by the GoogleAuthorizationFlow api client.
Raw
Readme.md
  • All Channels have a one to one mapping with the entry in the credentialStore.
  • The refreshing process is managed by the GoogleAuthorizationFlow api client.
Raw
YoutubeDataAPIHelperService.java
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.brightcove.bumblebee.domain.entity.Binding;
import com.brightcove.bumblebee.domain.entity.ErrorCode;
import com.brightcove.bumblebee.domain.entity.RemoteMediaRelation;
import com.brightcove.bumblebee.domain.exception.
import com.brightcove.bumblebee.domain.origin.MediaDescription;
import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.auth.oauth2.CredentialStore;
import com.google.api.client.auth.oauth2.TokenResponse;
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
import com.google.api.client.googleapis.media.MediaHttpUploader;
import com.google.api.client.googleapis.media.MediaHttpUploaderProgressListener;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.InputStreamContent;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.youtube.YouTube;
import com.google.api.services.youtube.model.Video;
import com.google.api.services.youtube.model.VideoListResponse;
import com.google.api.services.youtube.model.VideoSnippet;
import com.google.api.services.youtube.model.VideoStatus;
import com.google.common.collect.Lists;
import com.yammer.metrics.Metrics;
import com.yammer.metrics.core.Counter;
@Service
public class YoutubeDataAPIHelperService {
public static Counter exceptionCounter = Metrics.newCounter(YoutubeDataAPIHelperService.class, "exceptionCount");
private static final Logger logger = LoggerFactory.getLogger(YoutubeDataAPIHelperService.class);
private static final String SCOPE = "https://www.googleapis.com/auth/youtube";
private static String VIDEO_FILE_FORMAT = "video/*";
private static String APPLICATION_NAME = "distribution-brightcove-com";
@Autowired CredentialStore credentialStore; // This is a database backed credentialStore implementation
@Autowired RemoteMediaRelationService remoteMediaRelationService; // This is a database object that represents the internal mediaId <-> youtubeId
@Autowired BumblebeeConfig config; // This is a systemwide configuration
@Autowired BindingService bindingService; // This represents a Youtube Channel realtion
@Autowired OriginFacade mOriginGateway; // THis is an interface to the internal media metadata service (provides media Information and video file url)
GoogleClientSecrets clientSecrets;
private GoogleAuthorizationCodeFlow getFlow() {
return new GoogleAuthorizationCodeFlow.Builder(new NetHttpTransport(),
new JacksonFactory(),
clientSecrets,
Lists.newArrayList(SCOPE))
.setCredentialStore(credentialStore)
.setAccessType("offline")
.setApprovalPrompt("force")
.build();
}
@PostConstruct
public void initialize() throws IOException {
Reader secrets_reader = new InputStreamReader(YoutubeDataAPIHelperService.class.getResourceAsStream("/client_secrets.json"));
this.clientSecrets = GoogleClientSecrets.load(new JacksonFactory(), secrets_reader);
}
Credential get_credential(String userId) throws IOException {
return this.getFlow().loadCredential(userId);
}
public String getAuthorizationUrl(String callbackUrl) {
return this.getFlow().newAuthorizationUrl()
.setState("google")
.setRedirectUri(callbackUrl)
.build();
}
public Credential authorize(String userId, String authorizationCode) throws IOException {
// Do a check to make sure this part was not already done by someone else
GoogleAuthorizationCodeFlow flow = this.getFlow();
TokenResponse response = flow.newTokenRequest(authorizationCode)
.setRedirectUri(config.getDashboardUrl())
.execute();
return flow.createAndStoreCredential(response, userId);
}
public Credential updateAuthorization(String userId, String authorizationCode) throws IOException {
// Do a check to make sure this part was not already done by someone else
TokenResponse response = this.getFlow().newTokenRequest(authorizationCode).setRedirectUri(config.getDashboardUrl()).execute();
this.credentialStore.delete(userId, null);
return this.getFlow().createAndStoreCredential(response, userId);
}
public RemoteMediaRelation delete(final long bindingId, final long mediaId) throws Exception {
RemoteMediaRelation rmr = remoteMediaRelationService.findRemoteMediaRelationByBindingAndMediaId(bindingId, mediaId);
try {
logger.info("Deleting Youtube Video: {}", mediaId);
if (rmr == null) {
logger.warn("Unable to find remote media relation for mediaId: {}, bindingId: {}", mediaId, bindingId);
return rmr;
}
if(rmr.getStatus() == RemoteMediaRelation.State.DELETED) {
logger.info("Remote media relation: {} already marked as deleted", rmr.getId());
return rmr;
}
if(rmr.getRemoteId() != null) {
this.getYoutube(rmr.getPublisherId(), rmr.getBindingId()).videos()
.delete(rmr.getRemoteId())
.setKey(config.getYoutubeApiKey())
.execute();
} else {
logger.info("Remote media id is null skipping youtube delete");
}
logger.info("YoutubeVideo deleted: {}", rmr.getRemoteId());
exceptionCounter.clear();
// Mark as deleted
return this.update_state(rmr.getId(), null, RemoteMediaRelation.State.DELETED);
} catch (Exception ex) {
exceptionCounter.inc();
if(rmr != null)
this.update_state(rmr.getId(), rmr.getRemoteId(), RemoteMediaRelation.State.ERROR_SYNC);
throw ex;
}
}
public RemoteMediaRelation upload(final long bindingId, final long mediaId) throws Exception {
final RemoteMediaRelation rmr = remoteMediaRelationService.findRemoteMediaRelationByBindingAndMediaId(bindingId, mediaId);
try {
logger.info("Uploading Video: {} to YouTube", mediaId);
if(rmr == null) {
logger.warn("Unable to find remote media relation: SKIPPING mediaId={}, bindingId={}", mediaId, bindingId);
return null;
}
if (rmr.getRemoteId() != null || rmr.getStatus() == RemoteMediaRelation.State.DELETED) {
logger.warn("Tried to upload a video that was already uploaded: Video ID: {} Youtube ID: {} or was set to Status: {}",
rmr.getMediaId(), rmr.getRemoteId(), rmr.getStatus());
return rmr;
}
if(rmr.getStatus() == RemoteMediaRelation.State.SYNCING) {
logger.warn("Attempted to upload mediaId: {}, bindingId: {} with state: SYNCING",
rmr.getMediaId(), rmr.getBindingId());
return rmr;
}
YouTube youtube = this.getYoutube(rmr.getPublisherId(), rmr.getBindingId());
MediaDescription media = getMediaDescription(rmr.getPublisherId(), rmr.getMediaId());
Video videoObjectDefiningMetadata = new Video();
// Set the video to 'public' may also be set as 'unlisted'
VideoStatus status = new VideoStatus();
status.setPrivacyStatus("public");
videoObjectDefiningMetadata.setStatus(status);
// We set a majority of the metadata with the VideoSnippet object.
VideoSnippet snippet = new VideoSnippet();
snippet.setTitle(truncateStringByByteCount(media.getName(), TITLE_MAX));
snippet.setDescription(truncateStringByByteCount(resolveDescription(bindingId, media), DESCRIPTION_MAX));
snippet.setTags(filterTagsForYoutube(media.getTags()));
videoObjectDefiningMetadata.setSnippet(snippet);
if (media.getAssetBCFSPath() == null) {
logger.warn("Asset BCFS Path for Media Id: {}, Asset Id: {} reported by origin is null ignoring task", rmr.getMediaId(), media.getAssetId());
return rmr;
}
InputStreamContent mediaContent;
if (media.getAssetBCFSPath().startsWith("http://")) {
mediaContent = new InputStreamContent(VIDEO_FILE_FORMAT, new BufferedInputStream(new URL(media.getAssetBCFSPath()).openStream()));
} else {
mediaContent = new InputStreamContent(VIDEO_FILE_FORMAT, new BufferedInputStream(new FileInputStream(media.getAssetBCFSPath())));
}
mediaContent.setLength(media.getAssetSize());
// The upload command includes:
// 1. Information we want returned after file is successfully uploaded.
// 2. Metadata we want associated with the uploaded video.
// 3. Video file itself.
YouTube.Videos.Insert videoInsert = youtube.videos().insert("snippet,status", videoObjectDefiningMetadata, mediaContent)
.setKey(config.getYoutubeApiKey());
// Set the upload type and add event listener.
MediaHttpUploader uploader = videoInsert.getMediaHttpUploader();
/*
* Sets whether direct media upload is enabled or disabled. True = whole media content is
* uploaded in a single request. False (default) = resumable media upload protocol to upload
* in data chunks.
*/
uploader.setDirectUploadEnabled(false);
MediaHttpUploaderProgressListener progressListener = new MediaHttpUploaderProgressListener() {
@Override
public void progressChanged(MediaHttpUploader uploader) throws IOException {
switch (uploader.getUploadState()) {
case INITIATION_STARTED:
tryUpdateProgress(rmr.getId(), RemoteMediaRelation.State.SYNCING, "Initiation Started");
break;
case INITIATION_COMPLETE:
tryUpdateProgress(rmr.getId(), RemoteMediaRelation.State.SYNCING, "Initiation Completed");
break;
case MEDIA_IN_PROGRESS:
tryUpdateProgress(rmr.getId(), RemoteMediaRelation.State.SYNCING, String.format("%.2f %% uploaded", (uploader.getProgress() * 100)));
break;
case MEDIA_COMPLETE:
tryUpdateProgress(rmr.getId(), RemoteMediaRelation.State.SYNCED, "Upload Complete");
break;
case NOT_STARTED:
tryUpdateProgress(rmr.getId(), RemoteMediaRelation.State.SYNCED, "Upload Not Started");
break;
}
}
};
uploader.setProgressListener(progressListener);
// Execute upload.
Video videoResponse = videoInsert.execute();
logger.info("Upload completed. mediaId: {} -> remoteId: {}", rmr.getMediaId(), videoResponse.getId());
// Update the remote_media_relation record
exceptionCounter.clear();
return this.update_rmr(rmr.getId(), videoResponse,
rmr.getPublisherId(), rmr.getBindingId(),
media.getId(), media.getVersion(),
RemoteMediaRelation.State.SYNCED);
} catch (Exception ex) {
exceptionCounter.inc();
if(rmr != null)
this.update_state(rmr.getId(), rmr.getRemoteId(), RemoteMediaRelation.State.ERROR_SYNC);
throw ex;
}
}
public RemoteMediaRelation update(final long bindingId, final long mediaId) throws Exception {
final RemoteMediaRelation rmr = remoteMediaRelationService.findRemoteMediaRelationByBindingAndMediaId( bindingId, mediaId);
try {
if(rmr == null) {
logger.warn("Unable to find remote media relation: SKIPPING mediaId={}, bindingId={}", mediaId, bindingId);
return null;
}
logger.info("Updating Video: {} to YouTube: {}", rmr.getMediaId(), rmr.getRemoteId());
YouTube youtube = this.getYoutube(rmr.getPublisherId(), rmr.getBindingId());
if(rmr.getRemoteId() == null) {
// try upload
logger.warn("Unable to update/edit unknown remote video, mediaId={} bindingId={}. Attempting to upload.", rmr.getMediaId(), rmr.getBindingId());
return this.upload(bindingId, mediaId);
}
this.update_state(rmr.getId(), rmr.getRemoteId(), RemoteMediaRelation.State.SYNCING);
// Create the video list request
YouTube.Videos.List listVideosRequest = youtube.videos().list("snippet,status")
.setId(rmr.getRemoteId())
.setKey(config.getYoutubeApiKey())
.setMaxResults(1l);
// Request is executed and video list response is returned
VideoListResponse listResponse = listVideosRequest.execute();
List<Video> videoList = listResponse.getItems();
Video video = videoList.isEmpty() ? null : videoList.get(0);
if(video == null) {
logger.warn("Remote Video not available on YouTube: mediaId={}, remoteId={}", rmr.getRemoteId(), rmr.getMediaId());
return rmr;
}
VideoStatus status = video.getStatus();
if(status!=null) {
if(StringUtils.equalsIgnoreCase(status.getUploadStatus(), "failed")) {
logger.warn("Video upload status: [failed] reason: [{}]", status.getFailureReason());
return this.tryUpdateProgress(rmr.getId(), RemoteMediaRelation.State.ERROR_SYNC, "Video upload status: [failed] reason: ["+status.getFailureReason()+"]");
}
if(StringUtils.equalsIgnoreCase(status.getUploadStatus(), "rejected")) {
logger.warn("Video upload status: [rejected] reason: [{}]", status.getRejectionReason());
return this.tryUpdateProgress(rmr.getId(), RemoteMediaRelation.State.ERROR_SYNC, "Video upload status: [failed] reason: ["+status.getRejectionReason()+"]");
}
}
MediaDescription media = getMediaDescription(rmr.getPublisherId(), rmr.getMediaId());
VideoSnippet snippet = video.getSnippet();
snippet.setTitle(truncateStringByByteCount(media.getName(), TITLE_MAX));
snippet.setDescription(truncateStringByByteCount(resolveDescription(bindingId, media), DESCRIPTION_MAX));
snippet.setTags(filterTagsForYoutube(media.getTags()));
video.setStatus(null);
// Create the video update request
YouTube.Videos.Update updateVideosRequest = youtube.videos()
.update("snippet", video)
.setKey(config.getYoutubeApiKey());
// Request is executed and updated video is returned
Video videoResponse = updateVideosRequest.execute();
exceptionCounter.clear();
return this.update_rmr(rmr.getId(), videoResponse,
rmr.getPublisherId(), rmr.getBindingId(),
media.getId(), media.getVersion(),
RemoteMediaRelation.State.SYNCED);
} catch (Exception ex) {
exceptionCounter.inc();
if(rmr != null)
this.update_state(rmr.getId(), rmr.getRemoteId(), RemoteMediaRelation.State.ERROR_SYNC);
throw ex;
}
}
RemoteMediaRelation update_state(long rmrId, String remoteId, RemoteMediaRelation.State state) {
RemoteMediaRelation rmr = remoteMediaRelationService.findRemoteMediaRelationById(rmrId);
if(rmr != null) {
rmr.setStatus(state);
rmr.setRemoteId(remoteId);
rmr.setProgress(state.getDescription());
rmr = remoteMediaRelationService.updateRemoteMediaRelation(rmr);
}
return rmr;
}
RemoteMediaRelation update_rmr(long remoteMediaRelationID, Video returnedVideo,
long publisherId, long bindingId, long mediaId, int mediaVersion,
RemoteMediaRelation.State state) {
RemoteMediaRelation remoteMediaRelation = remoteMediaRelationService.findRemoteMediaRelationById(remoteMediaRelationID);
remoteMediaRelation.setPublisherId(publisherId);
remoteMediaRelation.setBindingId(bindingId);
remoteMediaRelation.setMediaId(mediaId);
remoteMediaRelation.setMediaVersion(mediaVersion);
if(returnedVideo != null) {
remoteMediaRelation.setRemoteId(returnedVideo.getId());
remoteMediaRelation.setRemoteVersion(returnedVideo.getEtag());
remoteMediaRelation.setName(returnedVideo.getSnippet().getTitle());
}
remoteMediaRelation.setLastSyncedDate(new Date());
remoteMediaRelation.setStatus(state);
remoteMediaRelation.setProgress(state.getDescription());
remoteMediaRelation.setRetryCount(0);
return remoteMediaRelationService.updateRemoteMediaRelation(remoteMediaRelation);
}
YouTube getYoutube(long publisherId, long bindingId) {
String userId = publisherId + "_" + bindingId;
HttpRequestInitializer credential;
credential = this.get_credential(userId);
return new YouTube.Builder(new NetHttpTransport(), new JacksonFactory(), credential)
.setApplicationName(APPLICATION_NAME)
.build();
}
RemoteMediaRelation tryUpdateProgress(long rmr_id, RemoteMediaRelation.State state, String progress) {
logger.info("RemoteMediaRelation: {} State: {} Progress: {}", rmr_id, state, progress);
RemoteMediaRelation rmr = remoteMediaRelationService.findRemoteMediaRelationById(rmr_id);
rmr.setStatus(state);
rmr.setProgress(progress);
return remoteMediaRelationService.updateRemoteMediaRelation(rmr);
}
static final int TITLE_MAX = 100;
static final int DESCRIPTION_MAX = 5000;
static final int TAG_MIN = 2;
static final int TAG_MAX = 30;
static final int TAGS_MAX_LENGTH = 500;
static final String TAG_COMMA = ",";
static final Charset UTF8 = Charset.forName("UTF-8");
static final int TAG_COMMA_NUM_BYTES = TAG_COMMA.getBytes(UTF8).length;
static final String DEFAULT_TAG = "brightcove";
private List<String> filterTagsForYoutube(List<String> pMediaTags) {
List<String> allTags = Lists.newArrayList(DEFAULT_TAG);
if (pMediaTags != null) {
int byteCount = 0;
for(String tag: pMediaTags) {
if(tag == null) {
continue;
}
// ignore tags containing '<' or '>':
if (tag.contains("<") || tag.contains(">")) {
continue;
}
// Get the tag's length in bytes, which may be different from its length in
// characters.
int numTagBytes = tag.getBytes(UTF8).length;
// Skip tags that are too long or too short.
if ((numTagBytes < TAG_MIN) || (numTagBytes > TAG_MAX)) {
continue;
}
// Calculate what the new length would be, if we added this tag.
int newCount = byteCount + numTagBytes;
// A tag with spaces in it will have quotes added by YouTube, so we
// have to take that into account. Note that if we add the quotes
// ourselves, however, YouTube won't parse the request! Yay!
if (tag.contains(" ")) {
newCount += 2;
}
// Take into account the fact that we might also be adding a comma.
if (byteCount > 0) {
newCount += TAG_COMMA_NUM_BYTES;
}
// Skip any tag that would push the overall length over the limit
if (newCount > TAGS_MAX_LENGTH) {
continue;
}
// If we're here, then it's time to actually add the tag
allTags.add(tag);
byteCount = newCount;
}
}
return allTags;
}
private MediaDescription getMediaDescription(long publisherId, long mediaId) {
MediaDescription media;
try {
media = mOriginGateway.findMediaById(publisherId, mediaId);
} catch (Exception ex) {
logger.warn("Unable to get media information from origin mediaId: {} publisherId: {}", mediaId, publisherId);
throw new TemporaryDistributionException("Unable to get media metadata, connectivity problems");
}
if(media == null) {
logger.warn("Received null media description from origin publisherId: {} mediaId: {}", publisherId, mediaId);
throw new TemporaryDistributionException("Unable to get media metadata");
}
return media;
}
/**
* Truncates strings by the byte count
* @param pString Target string
* @param bytecount max byte count
* @return Truncated string
*/
protected static String truncateStringByByteCount(final String pString, int bytecount) {
if(!org.springframework.util.StringUtils.hasLength(pString)) {
return "";
}
if(pString.getBytes(UTF8).length < bytecount) {
return pString;
}
// We need to check this character by character and count the number of bytes added
// and then see if the count goes above the needed bytecount, if that is the case we
// should just remove the last added character, we cannot do this just by byte count
// because that could cause some of the characters to be incomplete (half of their bytes missing).
StringBuilder sb = new StringBuilder();
for(char ch: pString.toCharArray()) {
sb.append(ch);
// if we exceeded the count remove the last character and break out.
if(sb.toString().getBytes().length > bytecount) {
sb.deleteCharAt(sb.length()-1);
break;
}
}
return sb.toString();
}
// Some publishers use the long description. See TEQUILA-344
private String resolveDescription(long bindingId, MediaDescription media) {
Binding b = bindingService.findBindingById(bindingId);
if (b.isUseLongDescription() && StringUtils.isNotBlank(media.getLongDescription())) {
return media.getLongDescription();
}
else {
return media.getDescription();
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment