View Single Post
Old 05-27-2017, 03:49 AM  
deonbell
Confirmed User
 
deonbell's Avatar
 
Industry Role:
Join Date: Sep 2015
Posts: 1,045
Best way to scrape this site?

Best way to scrape this site?

https://bitinfocharts.com/top-100-ri...addresses.html

I tried to scrape with beautiful soup. I got a connection refused. I may copy site local, then scrape using beautiful soup.

I want to keep an updated list of 10,000 bitcoin addresses. I manually entered the top 600 bitcoin addresses into a text file and uploaded to mysql.

I have my bitcoin collider working now. I decided that instead of searching every bitcoin address in the blockchain I want to search a smaller database locally.

This coding is so much fun.

Code:
#!/usr/bin/env python
# Joric/bitcoin-dev, june 2012, public domain

import hashlib
import time
import ctypes
import ctypes.util
import sys
import MySQLdb

ssl = ctypes.cdll.LoadLibrary (ctypes.util.find_library ('ssl') or 'libeay32')

def check_result (val, func, args):
    if val == 0: raise ValueError 
    else: return ctypes.c_void_p (val)

ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p
ssl.EC_KEY_new_by_curve_name.errcheck = check_result

class KEY:
    def __init__(self):
        NID_secp256k1 = 714
        self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1)
        self.compressed = False
        self.POINT_CONVERSION_COMPRESSED = 2
        self.POINT_CONVERSION_UNCOMPRESSED = 4

    def __del__(self):
        if ssl:
            ssl.EC_KEY_free(self.k)
        self.k = None

    def generate(self, secret=None):
        if secret:
            self.prikey = secret
            priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new())
            group = ssl.EC_KEY_get0_group(self.k)
            pub_key = ssl.EC_POINT_new(group)
            ctx = ssl.BN_CTX_new()
            ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx)
            ssl.EC_KEY_set_private_key(self.k, priv_key)
            ssl.EC_KEY_set_public_key(self.k, pub_key)
            ssl.EC_POINT_free(pub_key)
            ssl.BN_CTX_free(ctx)
            return self.k
        else:
            return ssl.EC_KEY_generate_key(self.k)

    def get_pubkey(self):
        size = ssl.i2o_ECPublicKey(self.k, 0)
        mb = ctypes.create_string_buffer(size)
        ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb)))
        return mb.raw

    def get_secret(self):
        bn = ssl.EC_KEY_get0_private_key(self.k);
        bytes = (ssl.BN_num_bits(bn) + 7) / 8
        mb = ctypes.create_string_buffer(bytes)
        n = ssl.BN_bn2bin(bn, mb);
        return mb.raw.rjust(32, chr(0))

    def set_compressed(self, compressed):
        self.compressed = compressed
        if compressed:
            form = self.POINT_CONVERSION_COMPRESSED
        else:
            form = self.POINT_CONVERSION_UNCOMPRESSED
        ssl.EC_KEY_set_conv_form(self.k, form)

def dhash(s):
    return hashlib.sha256(hashlib.sha256(s).digest()).digest()

def rhash(s):
    h1 = hashlib.new('ripemd160')
    h1.update(hashlib.sha256(s).digest())
    return h1.digest()

b58_digits = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'

def base58_encode(n):
    l = []
    while n > 0:
        n, r = divmod(n, 58)
        l.insert(0,(b58_digits[r]))
    return ''.join(l)

def base58_decode(s):
    n = 0
    for ch in s:
        n *= 58
        digit = b58_digits.index(ch)
        n += digit
    return n

def base58_encode_padded(s):
    res = base58_encode(int('0x' + s.encode('hex'), 16))
    pad = 0
    for c in s:
        if c == chr(0):
            pad += 1
        else:
            break
    return b58_digits[0] * pad + res

def base58_decode_padded(s):
    pad = 0
    for c in s:
        if c == b58_digits[0]:
            pad += 1
        else:
            break
    h = '%x' % base58_decode(s)
    if len(h) % 2:
        h = '0' + h
    res = h.decode('hex')
    return chr(0) * pad + res

def base58_check_encode(s, version=0):
    vs = chr(version) + s
    check = dhash(vs)[:4]
    return base58_encode_padded(vs + check)

def base58_check_decode(s, version=0):
    k = base58_decode_padded(s)
    v0, data, check0 = k[0], k[1:-4], k[-4:]
    check1 = dhash(v0 + data)[:4]
    if check0 != check1:
        raise BaseException('checksum error')
    if version != ord(v0):
        raise BaseException('version mismatch')
    return data

def gen_eckey(passphrase=None, secret=None, pkey=None, compressed=False, rounds=1, version=0):
    k = KEY()
    if passphrase:
        secret = passphrase.encode('utf8')
        for i in xrange(rounds):
            secret = hashlib.sha256(secret).digest()
    if pkey:
        secret = base58_check_decode(pkey, 128+version)
        compressed = len(secret) == 33
        secret = secret[0:32]
    k.generate(secret)
    k.set_compressed(compressed)
    return k

def get_addr(k,version=0):
    pubkey = k.get_pubkey()
    secret = k.get_secret()
    hash160 = rhash(pubkey)
    addr = base58_check_encode(hash160,version)
    payload = secret
    if k.compressed:
        payload = secret + chr(1)
    pkey = base58_check_encode(payload, 128+version)
    print ("address:---" + addr)
    db = MySQLdb.connect("localhost","root","haha","bitcoindb" )
    cursor = db.cursor()
    sql = "SELECT * FROM bitcointable \
       WHERE address = '%s'" % (addr)
    try:
     # Execute the SQL command
       cursor.execute(sql)
      # Fetch all the rows in a list of lists.
       results = cursor.fetchall()
       for row in results:
         print "we found one!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
         print addr
         print pkey
         winnerfile = open('top100winner.txt', 'w')
         winnerfile.write(pkey + ':' + addr)
         winnerfile.close() 
         exit()
    except:
          print addr
 
    db.close() 
    return addr, pkey

def reencode(pkey,version=0):
    payload = base58_check_decode(pkey,128+version)
    secret = payload[:-1]
    payload = secret + chr(1)
    pkey = base58_check_encode(payload, 128+version)
    print get_addr(gen_eckey(pkey))

def test(otherversion):
    # random compressed
    #print get_addr(gen_eckey(compressed=True,version=otherversion),version=otherversion)

    # uncomment these to create addresses via a different method
    # random uncompressed
    #print get_addr(gen_eckey())
    # by secret
    
    inputfile = open('lastsearch.txt', 'r')
    startdata = inputfile.read()
    inputfile.close() 
    print "starting point"

    counterfile = open('top100counter.txt', 'r')
    counter = counterfile.read()
    counterfile.close() 
   
    inputlove=startdata.strip()   
    inputlove = inputlove.zfill(64)
    inputkeyin = int(inputlove,16)

    startingpoint = int(inputlove,16)
    outcounter = int(counter)
    
    while inputkeyin < startingpoint + 10000: 
        print inputkeyin
        inputkeyhex = hex(inputkeyin)[2:-1]
   #     print inputkeyhex
        get_addr(gen_eckey(secret=inputkeyhex.decode('hex')))
        get_addr(gen_eckey(compressed=True,secret=inputkeyhex.decode('hex')))
        inputkeyin = int(inputkeyhex,16)
        inputkeyin = inputkeyin + 1
        outcounter = outcounter + 1

    outputfile = open('lastsearch.txt', 'w')
    outputfile.write(inputkeyhex)
    outputfile.close() 
    if outcounter > 0:
       outputcounter = open('top100counter.txt', 'w')
       stroutcounter=str(outcounter)
       outputcounter.write(stroutcounter)
       outputcounter.close() 
  
if __name__ == '__main__':
    import optparse
    parser = optparse.OptionParser(usage="%prog [options]")
    parser.add_option("--otherversion", dest="otherversion", default=0,
                    help="Generate address with different version number")
    (options, args) = parser.parse_args()


answeryes = "y"
answercapitalyes = "Y"

test(int(options.otherversion))
deonbell is offline   Share thread on Digg Share thread on Twitter Share thread on Reddit Share thread on Facebook Reply With Quote