【利用ガイド】アクセスログ提供機能

(省略)
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_ltsv-012.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_ltsv-013.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_ltsv-014.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_Itsv-015.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_ltsv-016.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_ltsv-017.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_Itsv-018.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_ltsv-019.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_Itsv-020.log.gz 2017/02/17 00:01:12
20170216/22/0123456789abcdef0123456789abcdef_2017021622_001_ltsv-021.log.gz 2017/02/17 00:01:12
(省略)

c-ip:198.51.100.1<%TAB%>vhost-id:0123456789abcdef0123456789abcdef<%TAB%> req-id:0123456789abcdef0123456789abcdef/1.2.3.4:12/12345/12345/12345/12345<%TAB%>c-user:-<%TAB%> date-time:[04/Dec/2017:10:01:42 +0900]<%TAB%>method:GET<%TAB%> origin-uri:http://origin.example.com/dir1/dir2/filename.ext?query_key=value<%TAB%> proto:HTTP/1.1<%TAB%>status:404<%TAB%>sc-bytes:495<%TAB%> c-referer:"http://referer.example.com/dir/file?query_key=value"<%TAB%> time-taken:0.001<%TAB%>length:225<%TAB%>cache-status:HIT/F<%TAB%> request:"GET https://hostname.example.com/dir1/dir2/filename.ext?query_key=value HTTP/1.1"<%TAB%> content-type:"text/html; charset=iso-8859-1"<%TAB%>x-forwarded:"198.51.100.2,unknown"<%TAB%> user-agent:"MobileSafari/604.1 CFNetwork/889.9 Darwin/17.2.0"<%TAB%> last-modified:"Fri, 24 Feb 2017 11:35:21 GMT"<%TAB%>etag:"¥x223-549451ff2a722¥x22"<%TAB%> vary:"Accept-Encoding, Accept-Language"<%TAB%>user-add-key:"|Vary;Accept-Encoding=gzip"<%TAB%> cache-key:0123456789abcdef0123456789abcdef<%TAB%>user-data:""

198.51.100.1 -- [04/Dec/2017:10:00:10 +0900] "GET http://origin.example.com/dir1/dir2/filename.ext?query_key=value HTTP/1.1" 200 123830 "http://referer.example.com/dir/file?query_key=value" 0.040 123328 MISS/S GET https://hostname.example.com/dir1/dir2/filename.ext?query_key=value 192.168.51.70:8080 "text/html" "198.51.100.2,unknown" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299"                

198.51.100.1 -- [04/Dec/2017:10:00:11 +0900] "GET http://origin.example.com/dir1/dir2/filename.ext?query_key=value HTTP/2.0" 304 170 "http://referer.example.com/dir/file?query_key=value" "Mozilla/5.0 (Linux; Android 7.1.1; 507SH Build/S1116) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36"

# ログインユーザー名
FTP_LOGIN_USERNAME="clNNN_name"

# ログインパスワード
FTP_LOGIN_PASSWORD="password"

# 対象 CDN ホストの VHOST ID
# 例）0123456789abcdef0123456789abcdef
TARGET_VHOST_ID="vhost_id"

# 対象日付
# 例）20170426
TARGET_DATE="yyyymmdd"

# ファイル取得
wget -nv ftp://cdnext-log-store.stream.ne.jp/any/${TARGET_VHOST_ID}/${TARGET_DATE}/ ¥
--user ${FTP_LOGIN_USERNAME} --password ${FTP_LOGIN_PASSWORD} -r -P ${TARGET_DATE}

# 取得ファイルログ行数確認
find ${TARGET_DATE}/ -type f -print0 | xargs -r0 zcat |wc -l

# ファイル結合
find ${TARGET_DATE}/ -type f | sort | xargs cat ${TARGET_DATE}.gz

# 結合後ファイルログ行数確認
zcat ${TARGET_DATE}.gz |wc -l

# 日時昇順でソート（LTSV 向け）
# zcat ${TARGET_DATE}.gz | sort -k5,5 sorted.log

# 日時昇順でソート（J-Stream 独自、Apache combined 向け）
# zcat ${TARGET_DATE}.gz | sort -k4,4 sorted.log

rem ログインユーザー名
set FTP_LOGIN_USERNAME=clNNN_name

rem ログインパスワード
set FTP_LOGIN_PASSWORD=password

rem 対象 CDN ホストの VHOST ID
rem 例）0123456789abcdef0123456789abcdef
set TARGET_VHOST_ID=vhost_id

rem 対象日付
set TARGET_DATE=yyyymmdd

rem ファイル取得
wget -nv ftp://cdnext-log-store.stream.ne.jp/any/%TARGET_VHOST_ID%/%TARGET_DATE%/ ^
--user %FTP_LOGIN_USERNAME% --password %FTP_LOGIN_PASSWORD% -r -P %TARGET_DATE%

rem 取得ファイルログ行数確認
dir /B /S *.gz | sed -e "s/¥¥/¥//g" | sort | xargs gzip -cd | wc -l

rem ファイル結合
dir /B /S *.gz | sed -e "s/¥¥/¥//g" | sort | xargs gzip -cd | gzip -c %TARGET_DATE%.gz

rem 結合後ファイルログ行数確認
gzip -cd %TARGET_DATE%.gz | wc -l