0x01 区块链浏览器爬虫

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
import requests
import time

def find_node_by_block_chain(chain='ether'):
pass

def announce_node_to_block_chain(ip, port, chain='ether'):
pass

block_list_url = 'https://api.yitaifang.com/index/largetxs/?page={page}&limit={limit}'
block_info_url = 'https://api.yitaifang.com/index/transactions/?page={page}&number={number}&limit={limit}'
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'}

LIMIT_NUM = 1000
REQ_NUM = 2000

# OPENDANMAKU 0x4f50454e44414e4d414b55204
# protocal name
PTC_NAME = '0x4f50454e44414e4d414b55'

# 9939615


req = requests.get(block_list_url.format(page=1, limit=20), headers=headers)
pages = req.json()['data']['result'][0]['blockNumber']
print(pages)

pages=9939630
node_list=[]

for block_num in range(pages, pages-10000, -1):
if(block_num%10==0):
print('Chicking block of %s... (%s nodes has been found.)'%(block_num, len(node_list)))
time.sleep(5)
req = requests.get(block_info_url.format(page=1,number=block_num,limit=LIMIT_NUM), headers=headers)
block = req.json()
for trans in block['data']['result']:
if(trans['input'][:24]==PTC_NAME):
print('Found node in block %s'%block_num)
print('Input: %s'%bytes.fromhex(trans['input'][2:]))
node_list.append(trans['input'])

exit()


#if(int(item['blockNumber'])%10==0):
print('Chicking block of %s...'%item['blockNumber'])
ii=1
req2 = requests.get(block_info_url.format(page=ii,number=item['blockNumber'],limit=LIMIT_NUM), headers=headers)
block = req2.json()
for trans in block['data']['result']:
if(trans['input'][:24]==PTC_NAME):
print('Found server in block %s'%item['blockNumber'])
print('Input: %s'%bytes.fromhex(trans['input']))

0x02 自己放到区块链上的一张图片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# file 2 hexstr
'''
with open("E:/Pictures/test2.jpg", "rb") as f:
data = f.read()
print(len(data))
print(hex(int.from_bytes(data[:20480], byteorder='big')))
'''


import requests
from PIL import Image
import time

headers={
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Host': 'ropsten.ethhelp.cn',
'Origin': 'http://ropsten.ethhelp.cn',
'Proxy-Connection': 'keep-alive',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
}

bc_url = 'http://ropsten.ethhelp.cn/tx'

transactions = [
'0x90420a0d29ec6675c2b6ae674c2dc7a56f338589984d2dc112c7cfbfba2c1072',
'0xc94e8f5db4f5af332ae6d8bee65b572746bfd345bcf58e7243471125450cc96f',
'0xee020f4659aa89aecd73b02b03b58eb81e661c0f82b5fe19f601daa23a4bcacc',
'0x9f4cd87a90e792a4c10229673c14e4fe49c3c3a028994ec3929fb589ee6ec323',
'0xb9eadf1dbc247e7273a096645fb34aeefa63f6b75c16449b49e9ef5d6c126346',
]

i=0
with open('test.png', 'wb') as f:
for trans in transactions:
req = requests.post(bc_url, headers=headers, data={'hash': trans})
f.write(bytes.fromhex(req.json()['data']['input'][2:]))
time.sleep(5)
i+=1
print('Downloading finished %s of %s'%(i, len(transactions)))

img = Image.open('test.png')

img.show()

0x03 python发送b站直播弹幕

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#encoding=utf-8
# Created by double lin at 2018/10/10
# https://blog.csdn.net/qq_32670879/article/details/83002369
import requests
import time
form_data = {
'color': '65532',
'fontsize': '25',
'mode': '1',
'msg': 'test33333333',
'rnd': int(time.time()),
'roomid': '1136753',
'csrf_token': 'cce335cbfa5bfd292a049b813175bd12',
'csrf': 'cce335cbfa5bfd292a049b813175bd12'
}
# 设置cookie值帮助我们在发送弹幕的时候,服务器识别我们的身份
cookie = {
}
res = requests.post('https://api.live.bilibili.com/msg/send', cookies=cookie, data=form_data)
print (res.status_code)

print (res.json())

0x04 JavaScript b站弹幕姬

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
const textEncoder = new TextEncoder('utf-8');
const textDecoder = new TextDecoder('utf-8');

const readInt = function(buffer, start, len) {
let result = 0
for (let i = len - 1; i >= 0; i--) {
result += Math.pow(256, len - i - 1) * buffer[start + i]
}
return result
}

const writeInt = function(buffer, start, len, value) {
let i = 0
while (i < len) {
buffer[start + i] = value / Math.pow(256, len - i - 1)
i++
}
}

const encode = function(str, op) {
let data = textEncoder.encode(str);
let packetLen = 16 + data.byteLength;
let header = [0, 0, 0, 0, 0, 16, 0, 1, 0, 0, 0, op, 0, 0, 0, 1]
writeInt(header, 0, 4, packetLen)
return (new Uint8Array(header.concat(...data))).buffer
}
const decode = function(blob) {
return new Promise(function(resolve, reject) {
let reader = new FileReader();
reader.onload = function(e) {
let buffer = new Uint8Array(e.target.result)
let result = {}
result.packetLen = readInt(buffer, 0, 4)
result.headerLen = readInt(buffer, 4, 2)
result.ver = readInt(buffer, 6, 2)
result.op = readInt(buffer, 8, 4)
result.seq = readInt(buffer, 12, 4)
if (result.op === 5) {
result.body = []
let offset = 0;
while (offset < buffer.length) {
let packetLen = readInt(buffer, offset + 0, 4)
let headerLen = 16 // readInt(buffer,offset + 4,4)
let data = buffer.slice(offset + headerLen, offset + packetLen);
let body = textDecoder.decode(data);
if (body) {
result.body.push(JSON.parse(body));
}
offset += packetLen;
}
} else if (result.op === 3) {
result.body = {
count: readInt(buffer, 16, 4)
};
}
resolve(result)
}
reader.readAsArrayBuffer(blob);
});
}
const ws = new WebSocket('wss://broadcastlv.chat.bilibili.com:2245/sub');
ws.onopen = function() {
ws.send(encode(JSON.stringify({
roomid: 5322
}), 7));
};
// 如果使用的是控制台,这两句一定要一起执行,否侧onopen不会被触发
setInterval(function() {
ws.send(encode('', 2));
}, 30000);
ws.onmessage = async function(msgEvent) {
const packet = await decode(msgEvent.data);
switch (packet.op) {
case 8:
console.log('加入房间');
break;
case 3:
const count = packet.body.count
console.log(`人气:${count}`);
break;
case 5:
packet.body.forEach((body) => {
switch (body.cmd) {
case 'DANMU_MSG':
console.log(`${body.info[2][1]}: ${body.info[1]}`);
break;
case 'SEND_GIFT':
console.log(`${body.data.uname} ${body.data.action} ${body.data.num} 个 ${body.data.giftName}`);
break;
case 'WELCOME':
console.log(`欢迎 ${body.data.uname}`);
break;
// 此处省略很多其他通知类型
default:
console.log(body);
}
})
break;
default:
console.log(packet);
}
};

0x05 b站收藏夹自动下载(未完工)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import requests
import time
import sys
import you_get
import pymysql

def download(url, path):
sys.argv = ['you-get', '--playlist', '-o', path, url]
you_get.main()

#if __name__ == '__main__':
# 视频网站的地址
#url = 'https://www.bilibili.com/bangumi/play/ep118488?from=search&seid=5050973611974373611'

# 视频输出的位置
#path = 'G:/test'
#download(url, path)
sql = 'insert into fav_list(avid,title,author,partcount,cover) values(%s,%s,%s,%s,%s);'

def first_use():
favid=46691401
params={
'media_id': '929964401',
'pn': '1',
'ps': '20',
'keyword': '',
'order': 'mtime',
'type': '0',
'tid': '0',
'jsonp': 'jsonp'
}
req_data = requests.get('https://api.bilibili.com/medialist/gateway/base/spaceDetail', params)
print(req_data.url)
print(req_data.json()['code'])
data=req_data.json()
page_num=(data['data']['info']['media_count']-1)//20+2
for page in range(1,page_num):
params['pn']=page
req_data = requests.get('https://api.bilibili.com/medialist/gateway/base/spaceDetail', params)
for video in req_data.json()['data']['medias']:
print(video['id'],end=' ')
print(video['title'])
sqldata=[]
sqldata.append(video['id'])
sqldata.append(video['title'])
sqldata.append(video['upper']['name'])
sqldata.append(video['page'])
sqldata.append(video['cover'])
#print(sqldata)
#发送请求
image = requests.get(video['cover'])
with open('F:/favdown/'+str(video['id'])+'.jpg',"wb")as f:
f.write(image.content)
download('https://www.bilibili.com/video/av'+str(video['id']),'F:/favdown/av'+str(video['id'])+'.flv')
exit()
#data['data']['info']['media_count']==233
#data['data']['medias'][0]['cover']==xxx
#cover/id:av/title/upper-name
#avid,title,cover/favid/author/partcount/date

conn = pymysql.connect(
host='114.67.107.192',
port=3306,
user='root',
password='',
database='favdown',
charset='utf8'
)
# 获取一个光标
cursor = conn.cursor()
first_use()