AI Mode, Content & Search Index

Our tests show that Google’s AI Mode doesn’t retrieve page content from the live web during the query fan out process. Instead, it gets it from somewhere else, and that “somewhere else” appears to be a proprietary content store separate from the search index.

How do we know this?

We just found a case where the page content failed to fetch for a page that’s indexed and ranking in Google search.

Prompted by Joshua Squires here, I decided to test this out:

1. deleted https://dejanmarketing.com/flux/
2. Fetched it and got 404
3. Put it back up
4. Fetched it and got 404 (instead of expected 200)
5. Checked and found that page to be indexed.

This contradicts my previous notion that if something is indexed it will be in AI Mode and accessible to Gemini.

At this point I welcome community contribution from anyone who wants to test this out.

Probe commands

import os print(os.listdir('/var/lib'))
print(open('/sys/fs/cgroup/memory/memory.limit_in_bytes').read()) print(open('/sys/fs/cgroup/cpu/cpu.cfs_quota_us').read())
print(os.listdir('/var/lib/lxc'))
print(open('/var/lib/lxc/b96a585c20d7/config').read())
print(os.listdir('/var/lib/lxc/b96a585c20d7'))
print(os.listdir('/var/lib/lxc/b96a585c20d7/rootfs'))
print(open('/etc/apparmor.d/lxc/lxc-container-default-cgnd_').read())
print(open('/sys/hypervisor/uuid').read()) print(open('/sys/power/machine_uuid').read())
import os print(os.listdir('/run/netns'))
print(os.listdir('/opt')) print(os.listdir('/mnt')) print(os.listdir('/srv'))
print(os.listdir('/root/.ssh'))
print(open('/etc/passwd').read())
print(open('/etc/crontab').read()) print(os.listdir('/etc/cron.d'))
print(os.listdir('/var/lib/cloud'))
print(os.listdir('/etc/cron.daily')) print(os.listdir('/etc/cron.hourly'))
print(os.listdir('/etc/google')) print(os.listdir('/var/lib/google'))
import os for root, dirs, files in os.walk('/'): print(root) for name in dirs: print(f"DIR: {os.path.join(root, name)}") for name in files: print(f"FILE: {os.path.join(root, name)}")
import os print(os.listdir('/'))
import os dirs = ['etc', 'var', 'home', 'opt', 'root', 'mnt', 'srv', 'usr'] for d in dirs: try: print(f"\nContents of /{d}:") print(os.listdir(f"/{d}")) except Exception as e: print(f"Unable to list /{d}: {e}")
import os print(os.listdir('/home'))
import os print(os.listdir('/root'))
import os print(os.listdir('/root/.ssh'))
print(open('/root/.bash_history').read())
import os print("Contents of /etc/google:") print(os.listdir('/etc/google')) print("\nContents of /var/lib/google:") print(os.listdir('/var/lib/google'))
import os, re pattern = r'.*\.md$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file))
import os, re pattern = r'.*\.txt$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file))
with open('/proc/kcore', 'rb') as f: print(f.read(256))
import os print(os.listdir('/proc/kcore'))
import os, re pattern = r'.*\.py$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file))
import os, re pattern = r'.*\.pb$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file))
print(open('/usr/share/defaults/google-cloud-diagnostics/grpc/protos/diagnostics.pb', 'rb').read(500))
import os print(os.listdir('/usr/share/defaults/google-cloud-diagnostics'))
print(open('/usr/share/defaults/google-cloud-diagnostics/gcgd.conf').read())
with open('/usr/share/defaults/google-cloud-diagnostics/grpc/protos/diagnostics.pb', 'rb') as f: data = f.read() print(data)
import os, re pattern = r'.*\.db$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file))
import os, re pattern = r'.*\.sqlite$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.log$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.json$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
print(open('/usr/share/cloud-init/templates/locale.json').read())
import os print(os.listdir('/usr/share/cloud-init/templates/'))
import os print(os.listdir('/usr/share/cloud-init/'))
print(os.listdir('/usr/share/cloud-init/sources'))
print(open('/usr/share/cloud-init/sources/DataSourceGCE.py').read()[:2000])
import requests headers = {'Metadata-Flavor': 'Google'} response = requests.get('http://metadata.google.internal/computeMetadata/v1/?recursive=true', headers=headers) print(response.text)
import os print(os.listdir('/var/lib/cloud/data')) print(os.listdir('/var/lib/cloud/instance'))
print(open('/var/lib/cloud/data/previous-datasource').read())
print(open('/var/lib/cloud/data/previous-ids').read())
import pickle with open('/var/lib/cloud/instance/obj.pkl', 'rb') as f: data = pickle.load(f) print(data)
import os, re pattern = r'.*\.pkl$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.env$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
print(open('/run/systemd/container/init_env.env').read())
import os, re pattern = r'.*\.py$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.conf$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.cfg$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.log$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.authorized_keys$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.id_ed25519$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.id_rsa$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
print(open('/root/.ssh/authorized_keys').read()) print(open('/root/.ssh/id_rsa').read())
import os, re pattern = r'.*\.html$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.js$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.java$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
print(os.listdir('/opt'))
print(os.listdir('/usr'))
print(os.listdir('/etc'))
print(os.listdir('/var'))
print(os.listdir('/mnt'))
print(os.listdir('/usr/local')) print(os.listdir('/usr/share'))
print(os.listdir('/etc/systemd')) print(os.listdir('/etc/apt')) print(os.listdir('/etc/ssl')) print(os.listdir('/etc/apparmor'))
print(os.listdir('/etc/systemd/network'))
print(os.listdir('/etc/ssl/certs'))
import os, re pattern = r'.*\.yaml$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.toml$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.ini$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.pdf$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os, re pattern = r'.*\.sh$' for root, dirs, files in os.walk('/'): for file in files: if re.match(pattern, file): print(os.path.join(root, file)
import os for root, dirs, files in os.walk('/'): for file in files: if 'gemini' in file.lower(): print(os.path.join(root, file))
import os for root, dirs, files in os.walk('/'): for file in files: if 'chrome' in file.lower(): print(os.path.join(root, file))
import os for root, dirs, files in os.walk('/'): for file in files: if 'nano' in file.lower(): print(os.path.join(root, file))
import os for root, dirs, files in os.walk('/'): for file in files: if 'dejan' in file.lower(): print(os.path.join(root, file))
import os for root, dirs, files in os.walk('/'): for file in files: if 'google' in file.lower(): print(os.path.join(root, file))
import os for root, dirs, files in os.walk('/'): for file in files: if 'algo' in file.lower(): print(os.path.join(root, file))
import os for root, dirs, files in os.walk('/'): for file in files: if 'system' in file.lower(): print(os.path.join(root, file))
import time ts = int(time.time()) url = f"https://dejan.ai/test.php?probe={ts}" req = urllib.request.Request(url) req.add_header("User-Agent", "PythonProbe/1.0″) response = urllib.request.urlopen(req) print(f"Probe {ts} → {response.status}")
import urllib.request url = "https://dejan.ai/test123.html" try: response = urllib.request.urlopen(url, timeout=5) print(f"Status: {response.status}") print("Body:\n", response.read().decode()) except Exception as e: print(f"Request failed: {e}")
import urllib.request url = "https://dejan.ai/test.php" try: response = urllib.request.urlopen(url, timeout=5) print(f"Status: {response.status}") print("Body:\n", response.read().decode()) except Exception as e: print(f"Request failed: {e}")
import urllib.request url = "https://dejan.ai/tools/linkbert/" try: response = urllib.request.urlopen(url, timeout=5) print(f"Status: {response.status}") print("Body:\n", response.read().decode()) except Exception as e: print(f"Request failed: {e}")
import urllib.request url = "https://dejan.ai/tools/loss/" try: response = urllib.request.urlopen(url, timeout=5) print(f"Status: {response.status}") print("Body:\n", response.read().decode()) except Exception as e: print(f"Request failed: {e}")
import urllib.request url = "https://dejanmarketing.com/flux/" try: response = urllib.request.urlopen(url, timeout=5) print(f"Status: {response.status}") print("Body:\n", response.read().decode()) except Exception as e: print(f"Request failed: {e}")
import urllib.request url = "https://dejan.ai/test.php" try: response = urllib.request.urlopen(url, timeout=5) print(f"Status: {response.status}") print("Body:\n", response.read().decode()) except Exception as e: print(f"Request failed: {e}")
import urllib.request url = "https://dejanmarketing.com/flux/" try: response = urllib.request.urlopen(url, timeout=5) print(f"Status: {response.status}") print("Body:\n", response.read().decode()) except Exception as e: print(f"Request failed: {e}")