Best way to scrape this site?
https://bitinfocharts.com/top-100-ri...addresses.html
I tried to scrape with beautiful soup. I got a connection refused. I may copy site local, then scrape using beautiful soup.
I want to keep an updated list of 10,000 bitcoin addresses. I manually entered the top 600 bitcoin addresses into a text file and uploaded to mysql.
I have my bitcoin collider working now. I decided that instead of searching every bitcoin address in the blockchain I want to search a smaller database locally.
This coding is so much fun.
Code:
#!/usr/bin/env python
# Joric/bitcoin-dev, june 2012, public domain
import hashlib
import time
import ctypes
import ctypes.util
import sys
import MySQLdb
ssl = ctypes.cdll.LoadLibrary (ctypes.util.find_library ('ssl') or 'libeay32')
def check_result (val, func, args):
if val == 0: raise ValueError
else: return ctypes.c_void_p (val)
ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p
ssl.EC_KEY_new_by_curve_name.errcheck = check_result
class KEY:
def __init__(self):
NID_secp256k1 = 714
self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1)
self.compressed = False
self.POINT_CONVERSION_COMPRESSED = 2
self.POINT_CONVERSION_UNCOMPRESSED = 4
def __del__(self):
if ssl:
ssl.EC_KEY_free(self.k)
self.k = None
def generate(self, secret=None):
if secret:
self.prikey = secret
priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new())
group = ssl.EC_KEY_get0_group(self.k)
pub_key = ssl.EC_POINT_new(group)
ctx = ssl.BN_CTX_new()
ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx)
ssl.EC_KEY_set_private_key(self.k, priv_key)
ssl.EC_KEY_set_public_key(self.k, pub_key)
ssl.EC_POINT_free(pub_key)
ssl.BN_CTX_free(ctx)
return self.k
else:
return ssl.EC_KEY_generate_key(self.k)
def get_pubkey(self):
size = ssl.i2o_ECPublicKey(self.k, 0)
mb = ctypes.create_string_buffer(size)
ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb)))
return mb.raw
def get_secret(self):
bn = ssl.EC_KEY_get0_private_key(self.k);
bytes = (ssl.BN_num_bits(bn) + 7) / 8
mb = ctypes.create_string_buffer(bytes)
n = ssl.BN_bn2bin(bn, mb);
return mb.raw.rjust(32, chr(0))
def set_compressed(self, compressed):
self.compressed = compressed
if compressed:
form = self.POINT_CONVERSION_COMPRESSED
else:
form = self.POINT_CONVERSION_UNCOMPRESSED
ssl.EC_KEY_set_conv_form(self.k, form)
def dhash(s):
return hashlib.sha256(hashlib.sha256(s).digest()).digest()
def rhash(s):
h1 = hashlib.new('ripemd160')
h1.update(hashlib.sha256(s).digest())
return h1.digest()
b58_digits = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
def base58_encode(n):
l = []
while n > 0:
n, r = divmod(n, 58)
l.insert(0,(b58_digits[r]))
return ''.join(l)
def base58_decode(s):
n = 0
for ch in s:
n *= 58
digit = b58_digits.index(ch)
n += digit
return n
def base58_encode_padded(s):
res = base58_encode(int('0x' + s.encode('hex'), 16))
pad = 0
for c in s:
if c == chr(0):
pad += 1
else:
break
return b58_digits[0] * pad + res
def base58_decode_padded(s):
pad = 0
for c in s:
if c == b58_digits[0]:
pad += 1
else:
break
h = '%x' % base58_decode(s)
if len(h) % 2:
h = '0' + h
res = h.decode('hex')
return chr(0) * pad + res
def base58_check_encode(s, version=0):
vs = chr(version) + s
check = dhash(vs)[:4]
return base58_encode_padded(vs + check)
def base58_check_decode(s, version=0):
k = base58_decode_padded(s)
v0, data, check0 = k[0], k[1:-4], k[-4:]
check1 = dhash(v0 + data)[:4]
if check0 != check1:
raise BaseException('checksum error')
if version != ord(v0):
raise BaseException('version mismatch')
return data
def gen_eckey(passphrase=None, secret=None, pkey=None, compressed=False, rounds=1, version=0):
k = KEY()
if passphrase:
secret = passphrase.encode('utf8')
for i in xrange(rounds):
secret = hashlib.sha256(secret).digest()
if pkey:
secret = base58_check_decode(pkey, 128+version)
compressed = len(secret) == 33
secret = secret[0:32]
k.generate(secret)
k.set_compressed(compressed)
return k
def get_addr(k,version=0):
pubkey = k.get_pubkey()
secret = k.get_secret()
hash160 = rhash(pubkey)
addr = base58_check_encode(hash160,version)
payload = secret
if k.compressed:
payload = secret + chr(1)
pkey = base58_check_encode(payload, 128+version)
print ("address:---" + addr)
db = MySQLdb.connect("localhost","root","haha","bitcoindb" )
cursor = db.cursor()
sql = "SELECT * FROM bitcointable \
WHERE address = '%s'" % (addr)
try:
# Execute the SQL command
cursor.execute(sql)
# Fetch all the rows in a list of lists.
results = cursor.fetchall()
for row in results:
print "we found one!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
print addr
print pkey
winnerfile = open('top100winner.txt', 'w')
winnerfile.write(pkey + ':' + addr)
winnerfile.close()
exit()
except:
print addr
db.close()
return addr, pkey
def reencode(pkey,version=0):
payload = base58_check_decode(pkey,128+version)
secret = payload[:-1]
payload = secret + chr(1)
pkey = base58_check_encode(payload, 128+version)
print get_addr(gen_eckey(pkey))
def test(otherversion):
# random compressed
#print get_addr(gen_eckey(compressed=True,version=otherversion),version=otherversion)
# uncomment these to create addresses via a different method
# random uncompressed
#print get_addr(gen_eckey())
# by secret
inputfile = open('lastsearch.txt', 'r')
startdata = inputfile.read()
inputfile.close()
print "starting point"
counterfile = open('top100counter.txt', 'r')
counter = counterfile.read()
counterfile.close()
inputlove=startdata.strip()
inputlove = inputlove.zfill(64)
inputkeyin = int(inputlove,16)
startingpoint = int(inputlove,16)
outcounter = int(counter)
while inputkeyin < startingpoint + 10000:
print inputkeyin
inputkeyhex = hex(inputkeyin)[2:-1]
# print inputkeyhex
get_addr(gen_eckey(secret=inputkeyhex.decode('hex')))
get_addr(gen_eckey(compressed=True,secret=inputkeyhex.decode('hex')))
inputkeyin = int(inputkeyhex,16)
inputkeyin = inputkeyin + 1
outcounter = outcounter + 1
outputfile = open('lastsearch.txt', 'w')
outputfile.write(inputkeyhex)
outputfile.close()
if outcounter > 0:
outputcounter = open('top100counter.txt', 'w')
stroutcounter=str(outcounter)
outputcounter.write(stroutcounter)
outputcounter.close()
if __name__ == '__main__':
import optparse
parser = optparse.OptionParser(usage="%prog [options]")
parser.add_option("--otherversion", dest="otherversion", default=0,
help="Generate address with different version number")
(options, args) = parser.parse_args()
answeryes = "y"
answercapitalyes = "Y"
test(int(options.otherversion))