From 5724267b818997fde012b70d6824b7a46ae2e7cf Mon Sep 17 00:00:00 2001 From: Antoine Rousseau Date: Mon, 26 Aug 2019 13:09:52 +0200 Subject: [PATCH] handle videos --- README.md | 5 ++- google.js | 2 +- start.js | 125 +++++++++++++++++++++++++++++++++++++----------------- 3 files changed, 92 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index de8a8f7..2c22129 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ Then create a `.env` file based on `example.env` (`cp {example,}.env`), and edit ./start.js +By default, it will process all photos and videos. If you want only photos or only videos, use the `MEDIA` environment variable. Set it to `photos` or `videos`, e.g.: + + MEDIA=photos ./start.js + ## Daemonize You can use [PM2](https://github.com/Unitech/pm2) @@ -28,5 +32,4 @@ You can use [PM2](https://github.com/Unitech/pm2) ## Limitations -- This script does not handle videos ([yet](https://github.com/antoinerousseau/flickr2google/pull/1)) - If your [Google storage](https://drive.google.com/settings/storage) is limited and you hit the limit, the Google API will return a "Bad Request". You must then either buy more storage, or go to your [Google Photos settings](https://photos.google.com/settings), choose "High Quality" and click "Recover storage". This will convert your uploads to [16 Megapixels compressed photos](https://support.google.com/photos/answer/6220791), which the API cannot do on the fly. Also, you can only convert once per day. diff --git a/google.js b/google.js index cb8f25e..6086d93 100644 --- a/google.js +++ b/google.js @@ -66,7 +66,7 @@ module.exports = async () => { { url: GOOGLE_API_ENDPOINT + "uploads", headers: { - // "Content-type": "application/octet-stream", + "Content-type": "application/octet-stream", // "Content-length": set by stream "X-Goog-Upload-File-Name": filename, "X-Goog-Upload-Protocol": "raw", diff --git a/start.js b/start.js index fb92132..a8d1d17 100755 --- a/start.js +++ b/start.js @@ -6,14 +6,16 @@ const googleConnect = require("./google") const { ALBUMS_PATH } = require("./constants") const { log, logError, readJson, writeJson, fileExists, mkdir } = require("./utils") +const media = process.env.MEDIA || "all" // "photos" or "videos" +const extras = "url_o,media,path_alias,original_format" +const per_page = 500 // max + const getAlbumPath = (id) => `${ALBUMS_PATH}/${id}.json` if (!fileExists(ALBUMS_PATH)) { mkdir(ALBUMS_PATH) } -const per_page = 500 // max - const main = async () => { const { flickr, user_id } = await flickrConnect() const { stream, post } = await googleConnect() @@ -31,45 +33,64 @@ const main = async () => { id: "NotInSet", }) - const albums_cache = {} + const memory = {} photosets.forEach((set) => { const path = getAlbumPath(set.id) if (fileExists(path)) { - albums_cache[set.id] = readJson(path) - if (set.photos && albums_cache[set.id].num_photos !== set.photos) { + memory[set.id] = readJson(path) + if (set.photos != null && memory[set.id].num_photos !== set.photos) { // number of photos has changed since last time, update: - albums_cache[set.id].num_photos = set.photos - writeJson(path, albums_cache[set.id]) + memory[set.id].num_photos = set.photos + writeJson(path, memory[set.id]) + } + if (set.videos != null && memory[set.id].num_videos !== set.videos) { + // number of videos has changed since last time, update: + memory[set.id].num_videos = set.videos + writeJson(path, memory[set.id]) } } else { - albums_cache[set.id] = { - title: set.title && set.title._content, + memory[set.id] = { + title: set.id === "NotInSet" ? "Not in a set" : set.title._content, flickr_set: set.id, google_album: null, num_photos: set.photos, + num_videos: set.videos, + total: set.photos + set.videos, + uploaded_photos: 0, + uploaded_videos: 0, done: [], } - writeJson(path, albums_cache[set.id]) + writeJson(path, memory[set.id]) } }) for (let i = 0; i < photosets.length; i++) { const photoset_id = photosets[i].id const path = getAlbumPath(photoset_id) - const data = albums_cache[photoset_id] - if (data.done.length === data.num_photos) { + const data = memory[photoset_id] + + if (media === "photos" && data.num_photos === data.uploaded_photos) { + continue + } + if (media === "videos" && data.num_videos === data.uploaded_videos) { + continue + } + if (media === "all" && data.num_photos + data.num_videos === data.uploaded_photos + data.uploaded_videos) { continue } - // FOR EACH PHOTOSET, RETRIEVE PHOTOS + const total_photos = data.num_photos == null ? "?" : data.num_photos + const total_videos = data.num_videos == null ? "?" : data.num_videos log( - `Processing "${data.title || photoset_id}" set ${i + 1}/${photosets.length};`, + `Fetching ${media} in "${data.title}" set ${i + 1}/${photosets.length};`, `Flickr id: ${photoset_id};`, - `Total: ${data.num_photos || "?"} photos` + `Total: ${total_photos} photos & ${total_videos} videos` ) + // FOR EACH PHOTOSET, RETRIEVE PHOTOS + let photoset let page = 0 do { @@ -77,16 +98,28 @@ const main = async () => { if (photoset_id === "NotInSet") { const { body } = await flickr.photos.getNotInSet({ // https://www.flickr.com/services/api/flickr.photos.getNotInSet.html - media: "photos", - extras: "url_o", - page, + media, + extras, per_page, + page, }) photoset = body.photos - photoset.title = "Photos not in a set" - if (!data.num_photos) { - data.num_photos = Number(photoset.total) + const totalNotInSet = Number(photoset.total) + + if (!totalNotInSet) { + continue + } + if (media === "photos" && data.num_photos == null) { + data.num_photos = totalNotInSet + writeJson(path, data) + } + if (media === "videos" && data.num_videos == null) { + data.num_videos = totalNotInSet + writeJson(path, data) + } + if (media === "all" && data.total == null) { + data.total = totalNotInSet writeJson(path, data) } } else { @@ -94,22 +127,25 @@ const main = async () => { // https://www.flickr.com/services/api/flickr.photosets.getPhotos.html photoset_id, user_id, - media: "photos", - extras: "url_o", - page, + media, + extras, per_page, + page, }) photoset = body.photoset } - const { title, photo: photos, pages } = photoset + const { photo: items, pages } = photoset - log(`Processing page ${page}/${pages} (${photos.length} photos);`, `Done ${data.done.length}`) + log( + `Processing page ${page}/${pages} (${items.length} ${media === "all" ? "items" : media});`, + media === "all" ? `Done ${data.done.length}` : "" + ) if (!data.google_album) { const albumRequest = { album: { - title, + title: data.title, }, } const { json: album } = await post("albums", albumRequest) @@ -118,22 +154,34 @@ const main = async () => { writeJson(path, data) } - for (let j = 0; j < photos.length; j++) { - const photo = photos[j] - if (data.done.includes(photo.id)) { + for (let j = 0; j < items.length; j++) { + const item = items[j] + if (data.done.includes(item.id)) { continue } + let url + let ext = `.${item.originalformat}` + if (item.media === "video") { + url = `https://www.flickr.com/photos/${item.pathalias}/${item.id}/play/orig/${item.originalsecret}/` + if (item.originalformat === "jpg") { + // Flickr inconsistency + ext = "" + } + } else { + url = item.url_o + } + // FOR EACH PHOTO, UPLOAD TO GOOGLE PHOTOS - const uploadToken = await stream(photo.url_o, `flickr_${photo.id}.jpg`) + const uploadToken = await stream(url, `flickr_${item.id}${ext}`) - const media = { + const mediaItem = { // https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate albumId: data.google_album, newMediaItems: [ { - description: photo.title, + description: item.title, simpleMediaItem: { uploadToken, }, @@ -143,18 +191,19 @@ const main = async () => { const { json: { newMediaItemResults: results }, status, - } = await post("mediaItems:batchCreate", media) + } = await post("mediaItems:batchCreate", mediaItem) if (status === 200) { - if (results.length === 1 && results[0].mediaItem) { - data.done.push(photo.id) + if (results.length === 1 && results[0].mediaItem && !results[0].status.code) { + data.done.push(item.id) + data[`uploaded_${item.media}s`]++ writeJson(path, data) log("Created media item @", results[0].mediaItem.productUrl) } else { - logError("Media Item creation status 200 OK but wrong response:", results) + logError("Media Item creation status 200 OK but wrong response:", results[0].status) } } else { - logError("Could not create media item", results[0]) + logError("Could not create media item", results[0].status) } } } while (page < photoset.pages)