Raw Source Toggle Dropdown

• Minified Source

  Alternate update check target.

book777 / Tumblr posts downloader

// ==UserScript==
// @name         Tumblr posts downloader
// @author       book777
// @namespace    niknikolia@yahoo.com
// @updateURL    https://pastebin.com/raw/UFNzeTuE
// @version      181209.1
// @description  Use keys: s - Download and remove posts, c - Start autodownload and remove posts, v - Stop autodownload, q - Go to top, a - Go to footer
// @match        https://*.tumblr.com/*
// @run-at       document-end
// @license      MIT
// @require      http://danml.com/js/download.js
// @grant        window.close
// ==/UserScript==

//todo support text

console.clear();

document.book777 = {
  keybinds: {
    runsingle: 83, // s
    runauto: 67, // c
    runstop: 86, // v
    gotop: 81, // q
    gofooter: 65, // a
  },
  schedule: {
    add: 1000, // [msec (1000 = 1 sec)] Autoadd files add to queue duration
    download: 2000, // [msec] Duration for move files from queue to active download
    rawFileClose: 300, // [msec] Close window timeout for raw files. If a page resets itself indefinitely, you must to increase value.
    filesPer: 4, // How many files will be taken from queue
    queueMax: 20, // Max files in queue
    _queue: [], // struct {url: string, name: string}
    _active: [], // struct {url: string, name: string}
    _checker: null,
    _adder: null,
  },
};

//todo mark posts as scanned
function scheduleChecker() {
  if (document.book777.schedule._checker === null) {

    document.book777.schedule._checker = setInterval(function () {
      // Move files from queue to active
      for (let i = 0; i < document.book777.schedule._queue.length &&
        document.book777.schedule._active.length < document.book777.schedule.filesPer; ++i) {

        document.book777.schedule._active.push(document.book777.schedule._queue.shift());
      }

      console.log(`[B] Scheduler: ${document.book777.schedule._queue.length} in queue, ${document.book777.schedule._active.length} downloading`);

      // Download all active
      while (document.book777.schedule._active.length > 0) {
        let file = document.book777.schedule._active.pop();
        getURLAndDownload(file.url, file.name);
      }

    }, document.book777.schedule.download);
  }
}

document.onkeyup = function (e) {
  // You can change the binds https://css-tricks.com/snippets/javascript/javascript-keycodes/

  if (e.keyCode === document.book777.keybinds.runsingle) {
    console.log("[B] Add files to download queue and remove posts");
    scheduleChecker();
    postsDowloadAndRemove();
  }

  if (e.keyCode === document.book777.keybinds.runauto) {
    console.log("[B] Start autodownload and remove posts");
    scheduleChecker();
    if (document.book777.schedule._adder === null) {
      document.book777.schedule._adder = setInterval(function () {
        postsDowloadAndRemove();
      }, document.book777.schedule.add);
    }
    else {
      console.error("[B] Autostart already running");
    }
  }

  if (e.keyCode === document.book777.keybinds.runstop) {
    console.log("[B] Stop autodownload and remove posts");
    clearInterval(document.book777.schedule._adder);
    document.book777.schedule._adder = null;
  }

  if (e.keyCode === document.book777.keybinds.gotop) {
    console.log("[B] Go to top");
    window.scrollTo(0, 0);
  }

  if (e.keyCode === document.book777.keybinds.gofooter) {
    console.log("[B] Go to footer");
    window.scrollTo(0, document.body.scrollHeight);
  }

}

function postsDowloadAndRemove() {
  if (document.book777.schedule._queue.length >= document.book777.schedule.queueMax) {
    console.log("[B] So many files in queue");
    return;
  }

  // Remove strange requests
  if (window.cedexis && window.cedexis.MP) {
    window.cedexis.MP = '';
  }

  let posts = document.getElementById("posts").getElementsByClassName("post"); //.querySelectorAll(".post")

  // Update and back to same position
  if (posts.length <= document.book777.schedule.queueMax) {
    let sy = window.scrollY;
    window.scrollTo(0, 0);
    window.scrollTo(0, document.body.scrollHeight);
    setTimeout(function () {
      window.scrollTo(0, sy);
    }, 200);
  }

  for (let post of posts) {
    if (document.book777.schedule._queue.length >= document.book777.schedule.queueMax) {
      console.log("[B] Stop adding files to queue");
      break;
    }

    if (post.classList.contains("new_post_buttons")) {
      // First block "post create"

    }
    else if (post.getAttribute('data-type') === 'photo' && tryGetPhoto(post) === true) { //post.classList.contains("is_photo")

    }
    else if (post.getAttribute('data-type') === 'photoset' && tryGetPhotoSet(post) === true) {

    }
    else if (post.getAttribute('data-type') === 'video' && tryGetVideo(post) === true) { //post.classList.contains("is_video") || post.classList.contains("is_direct_video")

    }
    else if (post.getAttribute('data-type') === 'regular') { // post.classList.contains("is_reblog") // Reblog must be last

      if (tryGetFigures(post) === true) {

      }
      else if (tryGetPhoto(post) === true) {

      }
      else if (tryGetPhotoSet(post) === true) {

      }
      else if (tryGetVideo(post) === true) {

      }
      else {
        console.error("[B] Unknown post format \\/\n", post);
      }
    }
    else {
      console.error("[B] Unknown post format \\/\n", post);
    }
  }
}

function getURLAndDownload(url, filename) {
  let xhr = new XMLHttpRequest();
  xhr.open('get', url);
  xhr.responseType = 'blob';

  xhr.onreadystatechange = function () {
    if (this.readyState == 4) { // ready
      if (this.status >= 200 && this.status < 400) {
        download(xhr.response, filename);

      }
      else if (this.status >= 400) {
        console.error(`[B] '${url}' have bad response code: ${this.status}`);

      }
      else { //if (this.status == 0 && xhr.response === null)
        console.log(`[B] open '${url}' in new window`);
        window.open(`${url}#name=${filename}`, '_blank')
        //debugger;
      }
    }
  };

  xhr.send();
}

function getURLAndDownloadFeature(url, filename) {
  let req = new Request(url); //, {mode: "no-cors"} https://developer.mozilla.org/en-US/docs/Web/API/Request/Request#Parameters

  fetch(req).
  catch(err => function () {}). //cross origin fix
  then(function (response) {
    //debugger;
    if (response.status >= 200 && response.status < 400) {
      console.log(`[B] File '${url}' has been downloaded. Status '${response.status}'`);
      return response.blob();

    }
    else if (response.status >= 400) {
      console.error(`[B] Bad response status ${response.status} for '${url}'`);

    }
    else {
      console.error(`[B] Open url in new tab '${url}'. Response:`, response);
      window.open(`${url}#name=${filename}`, '_blank'); //try to open raw file
    }

  }).
  then(function (data) {
    if (data !== undefined) {
      download(data, filename);
    }
  }).
  catch(error => function () {
    console.error('[B] Error:', error);
  });

  //console.log("#", myRequest);
}

function addToQueue(url, file) {
  document.book777.schedule._queue.push({
    url: url,
    name: file
  });
}

function tryGetPhoto(postElement) {
  let photos = postElement.getElementsByClassName("post_media_photo");
  let getSomething = false;

  for (let photo of photos) {
    //console.log("[B] get user post", photo.src);
    let extension = photo.src.split('.').pop();

    addToQueue(photo.src, postElement.getAttribute('data-id') + ".photo." + extension);

    getSomething = true;
  }

  if (getSomething === true) {
    postElement.remove();
    return true;
  }
  else {
    return false;
  }
}

function tryGetFigures(postElement) { //todo sometimes are videos
  let figures = postElement.getElementsByTagName("figure");
  //let figures = post.getElementsByClassName("tmblr-full");
  let getSomething = false;

  for (let figure of figures) {
    let photos = figure.getElementsByTagName("img");
    for (let photo of photos) {
      //console.log("[B] User repost", photo.src);
      let extension = photo.src.split('.').pop();

      addToQueue(photo.src, postElement.getAttribute('data-id') + ".reblog." + extension);

      getSomething = true;
    }
  }

  if (getSomething === true) {
    postElement.remove();
    return true;
  }
  else {
    return false;
  }
}

function tryGetPhotoSet(postElement) {
  let photosets = postElement.getElementsByClassName("photoset");
  let getSomething = false;

  for (let photoset of photosets) {
    let photos = photoset.getElementsByTagName("img");
    for (let photo of photos) {
      //console.log("[B] User repost", photo.src);
      let extension = photo.src.split('.').pop();

      addToQueue(photo.src, postElement.getAttribute('data-id') + ".photoset." + extension);

      getSomething = true;
    }
  }

  if (getSomething === true) {
    postElement.remove();
    return true;
  }
  else {
    return false;
  }
}

function tryGetVideo(postElement) {
  let getSomething = false;

  let sources = postElement.getElementsByTagName("source");
  for (let source of sources) {
    //console.log(`[B] Video ${source.src}`);
    let extension = source.getAttribute('type').split('/').pop();

    addToQueue(source.src, postElement.getAttribute('data-id') + ".video." + extension);

    getSomething = true;
  }

  let posters = postElement.getElementsByTagName("video");
  for (let poster of posters) {
    //console.log(`[B] Video poster ${source.src}`);
    let extension = poster.getAttribute('poster').split('.').pop();

    addToQueue(poster.getAttribute('poster'), postElement.getAttribute('data-id') + ".poster." + extension);
  }

  if (getSomething === true) {
    postElement.remove();
    return true;
  }
  else {
    return false;
  }
}

function checkStatusCode() {
  let xhr = new XMLHttpRequest();
  xhr.open('get', location.href);
  xhr.onloadend = function () {
    if (this.status >= 400) {
      window.close();
    }
  }
  xhr.send();
}
checkStatusCode();

function checkRawFile() {
  if (document.body.firstChild.nodeName === "#text") {
    console.log('[B] script initialization...');
    return
  }

  let fChild = document.body.firstChild.tagName.toLowerCase();

  if (fChild === "div") {
    return;
  }

  if (fChild === "video") {
    downloadAndClose(window.location.href,
      (getQueryVariable('name') !== false ? getQueryVariable('name') : window.location.href.split('/').pop()));
    document.body.firstChild.pause();
  }

  if (fChild === "img") {
    downloadAndClose(window.location.href,
      (getQueryVariable('name') !== false ? getQueryVariable('name') : window.location.href.split('/').pop()));
  }
}
checkRawFile();

function downloadAndClose(url, filename) {
  let xhr = new XMLHttpRequest();

  xhr.open('get', url);
  xhr.responseType = 'blob';

  xhr.onreadystatechange = function () {
    if (this.readyState == 4) { // ready
      if (this.status >= 200 && this.status < 400) {
        download(xhr.response, filename);
        setTimeout(function () {
          window.close();
        }, document.book777.schedule.rawFileClose); // fix infinity reload
      }
      else {
        console.error(`[B] ${url} have code ${this.status}`, xhr.response);
      }
    }
  };

  xhr.send();
}

function getQueryVariable(variable) {
  var query = window.location.hash.substring(1); //remove '#'
  var vars = query.split("&");
  for (var i = 0; i < vars.length; i++) {
    var pair = vars[i].split("=");
    if (pair[0] === variable) {
      return pair[1];
    }
  }
  return false;
}