# usage: bds [-p num] [-t tag] [-u user] tag
bds vim
bds -p 1280
bds -u jvs
bds -t plistbuddy
bds -t tar
bds -t ipfw -u jvs

#!/opt/local/bin/bash

# "batch download snippets" from http://codesnippets.joyent.com and
# convert them to text files using man textutil (available on Mac OS X 10.4 or later).
#
# Note: Old snippet versions will be automatically replaced by the downloaded snippets without a backup!
#      An alternative to man textutil is html2text, http://www.mbayer.de/html2text/ (which is available via MacPorts).
#
# Author: jv
# License: The MIT License, http://www.opensource.org/licenses/mit-license.php
# Copyright (c) 2008 jv
#
# cat /usr/local/bin/bds
#
# usage: bds [-p num] [-t tag] [-u user] tag


declare BaseURL='http://codesnippets.joyent.com'
declare download_dir="${HOME}/Desktop/Snippets"

# make sure there is no trailing slash
BaseURL="${BaseURL%/}"
download_dir="${download_dir%/}"

declare BasePostURL="${BaseURL}/posts/show"
declare BaseTagURL="${BaseURL}/tag"
declare BaseUserURL="${BaseURL}/user"

# make sure there is no trailing slash
BasePostURL="${BasePostURL%/}"
BaseTagURL="${BaseTagURL%/}"
BaseUserURL="${BaseUserURL%/}"


# man textutil
declare InputEncoding='utf-8'
declare OutputEncoding='utf-8'

export IFS=$' \t\n'


# function to download a single post specified by a post number: bds -p num
# cf. snippet, http://codesnippets.joyent.com/posts/show/1282

function snippet() {

   declare NL OPWD file outputfile postnum title url

   if [[ "${1//[[:digit:]]/}" != "" ]]; then echo "Argument error. No positive integer: ${1}"; return 1; fi

   postnum="${1}"
   url="${BasePostURL}/${postnum}"
   download_dir="${download_dir}/single-downloads"
   /bin/mkdir -p "${download_dir}"
   OPWD="${PWD}"
   cd "${download_dir}"
   /usr/bin/curl -L -O -s --max-time 25 "${url}" || exit 1    # download snippet web page
   file="${download_dir}/${url##*/}"
   trap '/bin/rm -f "${file}"; exit 0' 0 1 2 13 15

   # get title of downloaded web page
   #title="$(/usr/bin/sed -E -n -e '/<[tT][iI][tT][lL][eE]>/{s/^.*<[tT][iI][tT][lL][eE]>(.*)<\/[tT][iI][tT][lL][eE]>.*$/\1/p;q;}' "${file}" | \
   #         /usr/bin/sed -E -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title

   title="$(/usr/bin/egrep -m 1 -io '<title>.*</title>' "${file}" | /usr/bin/sed -E -e 's/^<title>[[:space:]]*|[[:space:]]*<\/title>$//g' \
             -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title


   title="${title//CodeSnippets:/}"
   title="${title//\//:}"
   title="${title// /_}"
   title="${title//[[:cntrl:]]/}"
   title="${title%"${title##*[!_]}"}"   # remove trailing underscores

   if [[ $title == '_CodeDrive_Snippets_courtesy_of_Peter_Coopers_handy_little_app' ]] || [[ -z "$title" ]]; then
      printf "\e[0K\e[31m%s\e[0m:  %s\n" "couldn't access" "${url}"
      /bin/rm "${file}"
      return 1
   fi

   outputfile="${download_dir}/${postnum}_${title}.txt"
   #outputfile="${download_dir}/${title}.txt"  # without post number prefix
   #outputfile="${outputfile//__/_}"  # uniq underscores

   printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "saved as" "${outputfile}"

   /usr/bin/textutil -output "${outputfile}" -convert txt -inputencoding "${InputEncoding}" -encoding "${OutputEncoding}" "${file}"
   /bin/rm "${file}"

   # escape backslashes
   # man bash 2>/dev/null | less -p 'Each command in a pipeline'
   #outputfile="$(printf "%q" "${outputfile}")"  # cf. help printf
   outputfile="${outputfile//\\/\\\\}"

   NL=$'\\\n'

cat <<EOF | /bin/ed -s "${outputfile}"
H
,g/Snippets is a public source code repository/1,/Snippets is a public source code repository/d
,g/You need to create an account or log in to post comments to this site//You need to create an account or log in to post comments to this site/,\$d
,g|(See related posts)$|s|.See related posts.|${NL}${NL}|
,g|^to.* by.* on .*[[:digit:]]$|s|^to\(.*\) by\(.*\) on \(.*[[:digit:]]\)$|${NL}${NL}Author:\2${NL}Date: \3${NL}URL: ${url}${NL}Tags:\1${NL}|
,g|^Comments on this post$|s|\(Comments on this post\)|${NL}\1:|
,g| posts on .* at |s|\(.* posts on .* at .*\)|${NL}\1:|
w
EOF

# additional ed commands
# delete line numbers
# ,g|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}|s|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}\(.*\)$|\1|
# delete range of lines
# 4,11d


   cd "${OPWD}"
   return 0

}



#----------------------------------------- end of function snippet



declare pflag tflag uflag
declare cnt count dir_name file no_posts_check NL OPWD outputfile postnum tagsite title url urls website 

if [[ $# -eq 0 ]]; then 
   printf "%s\n%s\n" 'No arguments given!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1
fi


while getopts ":p:t:u:" option
do
  case $option in
    p) pflag="$OPTARG" ;;
    t) tflag="$OPTARG" ;;
    u) uflag="$OPTARG" ;;
    [?]) printf "%s\n%s\n" 'Argument error!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2; exit 1;;
    *) ;;
  esac
done

shift $(($OPTIND - 1))


if [[ $# -eq 1 ]]; then

   dir_name="${1}"
   tagsite="${BaseTagURL}/${1}"

elif [[ $# -gt 1 ]]; then

   printf "%s\n%s\n" 'Too many arguments!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1

elif [[ -n "${pflag}" ]]; then
   snippet "${pflag}"
   exit 0

elif [[ -n "${tflag}" ]] && [[ -n "${uflag}" ]]; then

   dir_name="${tflag}-${uflag}"
   tagsite="${BaseUserURL}/${uflag}/tag/${tflag}"

elif [[ -n "${tflag}" ]]; then

   dir_name="${tflag}"
   tagsite="${BaseTagURL}/${tflag}"

elif [[ -n "${uflag}" ]]; then

   dir_name="${uflag}"
   tagsite="${BaseUserURL}/${uflag}"

else

   printf "%s\n%s\n" 'Argument error!' "Usage: ${0##*/} [-p num] [-t tag] [-u user] tag" 1>&2
   exit 1

fi


tagsite="${tagsite%/}"

#echo $dir_name
#echo $tagsite

count=1
cnt=0
curl_max_time=20
website=''
no_posts_check=''
NL=$'\\\n'
download_dir="${download_dir}/${dir_name//\//:}"
download_dir="${download_dir%/}"
/bin/mkdir -p "${download_dir}"
OPWD="${PWD}"
cd "${download_dir}"

# print download directory
printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "download directory" "${download_dir}"


while [[ -z "${no_posts_check}" ]]; do

   # download website of the form: 
   # http://somewebsite.com/tag/bash/1,
   # http://somewebsite.com/user/name/1 or 
   # http://somewebsite.com/user/name/tag/bash/1

   website="$(/usr/bin/curl -L -s --max-time $curl_max_time "${tagsite}/${count}" )"

   if [[ $? -ne 0 ]]; then 
      printf "\e[0K\e[31m%s\e[0m:  %s\n" "curl_max_time ${curl_max_time}" "${tagsite}/${count}"
      exit 1
   fi

   #if [[ -n "$(printf "%s" "${website}" | /usr/bin/egrep -o 'Application error \(Apache\)')" ]]; then 
      #no_posts_check='Application error (Apache)'
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   #fi

   if [[ -n "$(printf "%s" "${website}" | /usr/bin/egrep -o '>No posts<')" ]]; then 
      no_posts_check='>No posts<'
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   fi

: <<-'COMMENT'

   # works for Bash 3.0 or later
   if [[ "${website}" =~ '>No posts<' ]]; then 
      no_posts_check="${BASH_REMATCH[0]}"
      #printf "\e[0K\e[31m%s\e[0m:  %s\n" "no further posts" "${no_posts_check}"
   fi

COMMENT


   if [[ -z "${no_posts_check}" ]]; then

      # extract relevant post URLs
      #urls=( $(printf "%s\n" "${website}" | /usr/bin/sed -E -n -e "s|^.* href=\"(/posts/show/[[:digit:]]+)\".*$|${BaseURL}\1|p;g") )
      urls=( $(printf "%s\n" "${website}" | /usr/bin/egrep -o 'href="/posts/show/[[:digit:]]+"' | /usr/bin/sed -E -n -e "s|href=\"(/posts/show/[[:digit:]]+)\"|${BaseURL}\1|p;g") )

      for ((i=0; i < "${#urls[@]}"; i++)); do

         url="${urls[${i}]}"

         postnum="${url##*/}"
         file="${download_dir}/${postnum}"
         trap '/bin/rm -f "${file}"; exit 0' 0 1 2 13 15

         /usr/bin/curl -L -O -s --max-time $curl_max_time "${url}"

         if [[ $? -ne 0 ]]; then 
            printf "\e[0K\e[31m%s\e[0m:  %s\n" "curl_max_time ${curl_max_time}" "${url}"
            continue
         fi
 

         # get title of downloaded web page
         #title="$(/usr/bin/sed -E -n -e '/<[tT][iI][tT][lL][eE]>/{s/^.*<[tT][iI][tT][lL][eE]>(.*)<\/[tT][iI][tT][lL][eE]>.*$/\1/p;q;}' "${file}" | \
         #    /usr/bin/sed -E -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title

         title="$(/usr/bin/egrep -m 1 -io '<title>.*</title>' "${file}" | /usr/bin/sed -E -e 's/^<title>[[:space:]]*|[[:space:]]*<\/title>$//g' \
                -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title


         title="${title//CodeSnippets:/}"
         title="${title//\//:}"
         title="${title// /_}"
         title="${title//[[:cntrl:]]/}"
         title="${title%"${title##*[!_]}"}"   # remove trailing underscores

         #printf "%s\n" "${title}"

         if [[ $title == '_CodeDrive_Snippets_courtesy_of_Peter_Coopers_handy_little_app' ]] || [[ -z "$title" ]]; then
            printf "\e[0K\e[31m%s\e[0m:  %s\n" "couldn't access" "${url}"
            /bin/rm "${file}"
            continue
         fi

         outputfile="${download_dir}/${postnum}_${title}.txt"
         #outputfile="${download_dir}/${title}.txt"  # without post number prefix
         #outputfile="${outputfile//__/_}"  # uniq underscores

         let cnt++
         printf "\e[0K\e[1;32m%-6s\e[0m  %s\n" "${cnt}" "${outputfile##*/}"

         /usr/bin/textutil -output "${outputfile}" -convert txt -inputencoding "${InputEncoding}" -encoding "${OutputEncoding}" "${file}"

         /bin/rm "${file}"


         # escape backslashes
         # man bash 2>/dev/null | less -p 'Each command in a pipeline'
         #outputfile="$(printf "%q" "${outputfile}")"  # cf. help printf
         outputfile="${outputfile//\\/\\\\}"

# edit $outputfile in-place with man ed
# first delete lines at the beginning & end,
# then remove the string 'See related posts' and add some newlines with $NL,
# then convert the line 'to...by...on' to line 'Author:...', line 'Date:...', line 'URL:...' and line 'Tags:...'
# and finally the last two ed commands insert two further newlines with $NL

cat <<EOF | /bin/ed -s "${outputfile}"
H
,g/Snippets is a public source code repository/1,/Snippets is a public source code repository/d
,g/You need to create an account or log in to post comments to this site//You need to create an account or log in to post comments to this site/,\$d
,g|(See related posts)$|s|.See related posts.|${NL}${NL}|
,g|^to.* by.* on .*[[:digit:]]$|s|^to\(.*\) by\(.*\) on \(.*[[:digit:]]\)$|${NL}${NL}Author:\2${NL}Date: \3${NL}URL: ${url}${NL}Tags:\1${NL}|
,g|^Comments on this post$|s|\(Comments on this post\)|${NL}\1:|
,g| posts on .* at |s|\(.* posts on .* at .*\)|${NL}\1:|
w
EOF

# additional ed commands
# delete line numbers
# ,g|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}|s|^[[:space:]]*[[:digit:]]\{1,\}[[:space:]]\{1,3\}\(.*\)$|\1|
# delete range of lines
# 4,11d


      done  # for

      let count++

   fi

done   # while


   cd "${OPWD}"


exit 0

mvn eclipse:eclipse -Declipse.downloadSources=true

snippet 345
snippet 1268
snippet 1281
snippet 182
snippet 1282
snippet 1115
snippet 5
snippet 23444

# $ cat $HOME/.bashrc

function snippet() {

   declare NL OPWD download_dir outputfile postnum title url

   if [[ "${1//[[:digit:]]/}" != "" ]]; then echo "Argument error. No positive integer: ${1}"; return 1; fi

   postnum="${1}"
   url="http://codesnippets.joyent.com/posts/show/${postnum}"
   download_dir="${HOME}/Desktop/Snippets"
   download_dir="${download_dir%/}"
   #printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "download directory" "${download_dir}"
   /bin/mkdir -p "${download_dir}"
   OPWD="$PWD"
   cd "${download_dir}"
   /usr/bin/curl -L -O -s --max-time 25 "${url}" || exit 1    # download snippet
   file="${download_dir}/${url##*/}"

   title="$(/usr/bin/sed -E -n -e '/<[tT][iI][tT][lL][eE]>/{s/^.*<[tT][iI][tT][lL][eE]>(.*)<\/[tT][iI][tT][lL][eE]>.*$/\1/p;q;}' "${file}" | \
             sed -E -e 's/\[[^][:space:]]*\]//g')"    # delete [xxx] tag elements of title

   title="${title//CodeSnippets:/}"
   title="${title//\//:}"
   title="${title// /_}"
   title="${title//[[:cntrl:]]/}"
   title="${title%"${title##*[!_]}"}"   # remove trailing "_" characters

   if [[ $title == '_CodeDrive_Snippets_courtesy_of_Peter_Coopers_handy_little_app' ]] || [[ -z "$title" ]]; then
      printf "\e[0K\e[31m%s\e[0m:  %s\n" "couldn't access" "${url}"
      /bin/rm "${file}"
      return 1
   fi

   outputfile="${HOME}/Desktop/Snippets/${postnum}_${title}.txt"
   #outputfile="${outputfile//__/_}"  # uniq underscores

   printf "\n\e[0K\e[1;30m%s\e[0m:  %s\n\n" "saved as" "${outputfile}"

   /usr/bin/textutil -output "${outputfile}" -convert txt "${file}"
   /bin/rm "${file}"

   # escape backslashes
   # man bash 2>/dev/null | less -p 'Each command in a pipeline'
   #outputfile="$(printf "%q" "${outputfile}")"  # cf. help printf
   outputfile="${outputfile//\\/\\\\}"

   NL=$'\\\n'

cat <<EOF | /bin/ed -s "${outputfile}"
H
,g/Snippets is a public source code repository/1,/Snippets is a public source code repository/d
,g/You need to create an account or log in to post comments to this site//You need to create an account or log in to post comments to this site/,\$d
,g|(See related posts)$|s|.See related posts.|${NL}${NL}|
,g|^to.* by.* on .*[[:digit:]]$|s|^to\(.*\) by\(.*\) on \(.*[[:digit:]]\)$|${NL}${NL}Author:\2${NL}Date: \3${NL}URL: ${url}${NL}Tags:\1${NL}|
,g|^Comments on this post$|s|\(Comments on this post\)|${NL}\1:|
,g| posts on .* at |s|\(.* posts on .* at .*\)|${NL}\1:|
w
EOF

   cd "$OPWD"
   return 0

}

#! /usr/bin/env ruby

require 'open-uri'

$home_dir = `echo $HOME`.chomp
`mkdir #{$home_dir}/tmp`
`mkdir #{$home_dir}/.tmp`
$tmp_dir = $home_dir + "/.tmp"
Dir.chdir($tmp_dir)        # Where wget will download files before moving.

puts "Files will be stored under -> #{$home_dir}/tmp"

#-- Filters for others -- #

$otvali_re = /http:\/\/img.otvali.ru\/[0-9]{4}\/[0-9]{2}\/[a-z0-9_]+_[0-9]{1,2}\.jpg/
$friendz_re = /thumbs\/([a-z0-9_]+.jpg)/

# -- Superiorpics Filters -- #

$fake_link = /(http:\/\/img[0-9]+.image(shack|venue)+\.(us|com)+\/(my|img).php\?image=(\w+_?)+\.jpg)/
$shack_link = /(http:\/\/img[0-9]+.image(shack|venue)+\.(us|com)+\/(\w_?-?)+\/(\w_?-?)+\/(\w+_?)+\.jpg)/
$av_link = /src="((\w_?-?)+\/(\w_?-?)+\/(\w+_?)+\.jpg)/i

# -- Filters for HollywoodTuna -- #

$tuna_small = /(\/images\/(\w+_?)+\.jpg)/

def makeNewFolder
  puts "OK, where do you want to hide these? Or ENTER and I'll do it for you."
  answer = gets

  if answer !="\n"
    $new_folder_name = answer.gsub(/[\s\W]/, '_').chomp
  else
    $new_folder_name = File.basename($address, ".*").gsub(/[\s\W]/, '_')
  end

  $path_to_new_dir = $home_dir + "/tmp/" + $new_folder_name + "/"
  `mkdir #{$path_to_new_dir}`
  puts "Created: " + $new_folder_name
  puts "Path --> " + $path_to_new_dir
end

def cleanUp
  puts "Moving files to #{$path_to_new_dir}..."
  `mv #{$home_dir}/.tmp/*.jpg #{$path_to_new_dir}`
end

## Main Program ##

$pages = Array.new

puts "Enter URL:"
$address = gets.chomp

if $address =~ /otvali.ru/

  puts "Scanning page..."
  $pages = open($address).read.scan($otvali_re).delete_if { |x| x =~ /_s__/ } # Remove thumbnails. Their URLs contains "_s__"

elsif $address =~ /friends.kz/

  puts "Scanning page..."
  $pages = open($address).read.scan($friendz_re).flatten!.uniq
  $pages.collect! { |path| "http://friends.kz/uploads/posts/" + path }

elsif $address =~ /superiorpics/

  puts "Scanning page..."
  $pages = open($address).read.scan($fake_link).flatten!.delete_if { |i| i !~ /http/ }  # Remove all regex objects save the full URL
  $pages.collect! do |page|
    if page =~ /imageshack/
      puts "Resolving #{page}"
      open(page).read.scan($shack_link)[0][0]   # Returns the actual link buried under two arrays of Regex objects
    elsif page =~ /imagevenue/
      puts "Resolving #{page}"
      path_to_pic = open(page).read.scan($av_link)[0][0]
      av_direct = "http://" + URI.parse(page).host + "/" + path_to_pic
    end
  end

elsif $address =~ /(hollywoodtuna|tunaflix)/

  puts "Who do you want? ENTER to take'em all! Start with OMIT for negative search."
  search_string = gets.chomp.gsub(/ /, "|" ) # Prepare to feed into Regex
  tuna_want = Regexp.new(search_string)
  puts tuna_want
  puts "Scanning page..."
  $pages = open($address).read.scan($tuna_small).flatten!.delete_if { |i| i !~ /(th|tn|small)\.jpg$/ }
  $pages.collect! do |page|
        page.gsub!("images", "images/bigimages")
        page.gsub!(/(th|tn|small)\.jpg$/, "big.jpg")
        tuna_direct = URI.parse($address).host + page
      end
    
    if search_string =~ /^OMIT/
        $pages.delete_if { |i| i =~ tuna_want }
      else
        $pages.delete_if { |i| i !~ tuna_want }
    end
end

## Report total of found images, if nothing then exit the program.

$pages.uniq!

puts $pages

if $pages.length == 0
  puts "Yeah right. Nice try. Off you go!"
  exit
elsif $pages.length <= 20
  puts "What?! #{$pages.length}, is that all?"
elsif $pages.length >= 21
  puts "What're y'fuckin nuts??! #{$pages.length} !!!"
end

makeNewFolder

$threads = []

$down_count = 0

for pic_to_fetch in $pages
    $threads << Thread.new(pic_to_fetch) do |link|
      puts "Fetching #{link}"
      %x{wget --quiet --random-wait #{link}}
      $down_count += 1
      puts "Downloaded " + $down_count.to_s + "/" + $pages.length.to_s
  end 
end

$threads.each {|thr| thr.join }

dwn_actual = Dir.entries($tmp_dir).length-2

cleanUp

dwn_moved = Dir.entries($path_to_new_dir).length-2

if $pages.length == dwn_actual and dwn_actual == dwn_moved
  puts "All completed @ #{Time.now.hour}:#{Time.now.min}. See y'around, dirty boy."
elsif dwn_moved <= dwn_actual
  diff = dwn_actual - dwn_moved
  puts "#{diff} got lost in the moving, baby. Goodbye."
elsif dwn_actual <= $pages.length
  diff = $pages.length - dwn_actual
  puts "#{diff} got lost coming down. Goodbye."
end