import tokenize import io code = 'x = 42' tokens = list(tokenize.generate_tokens(io.StringIO(code).readline)) print(f'Found {len(tokens)} tokens')