Technical SEO for Developers: Beyond the Basics

Introduction

Technical SEO is where engineering meets marketing. As developers, we have direct control over the technical factors that determine search visibility. This guide covers what matters most.

Crawlability and Indexability

Robots.txt Configuration

# robots.txt
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /*?*sort=
Disallow: /*?*filter=

# Sitemap location
Sitemap: https://example.com/sitemap.xml

# Crawl delay for aggressive bots
User-agent: AhrefsBot
Crawl-delay: 10

XML Sitemap Generation

interface SitemapUrl {
  loc: string;
  lastmod?: string;
  changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
  priority?: number;
}

async function generateSitemap(): Promise<string> {
  const pages = await getPublishedPages();
  const posts = await getPublishedPosts();
  
  const urls: SitemapUrl[] = [
    // Static pages
    { loc: 'https://example.com/', changefreq: 'daily', priority: 1.0 },
    { loc: 'https://example.com/about', changefreq: 'monthly', priority: 0.8 },
    { loc: 'https://example.com/services', changefreq: 'weekly', priority: 0.9 },
    
    // Dynamic pages
    ...pages.map(page => ({
      loc: `https://example.com/${page.slug}`,
      lastmod: page.updatedAt.toISOString(),
      changefreq: 'weekly' as const,
      priority: 0.7,
    })),
    
    // Blog posts
    ...posts.map(post => ({
      loc: `https://example.com/blog/${post.slug}`,
      lastmod: post.updatedAt.toISOString(),
      changefreq: 'monthly' as const,
      priority: 0.6,
    })),
  ];

  return `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
${urls.map(url => `  <url>
    <loc>${url.loc}</loc>
    ${url.lastmod ? `<lastmod>${url.lastmod}</lastmod>` : ''}
    ${url.changefreq ? `<changefreq>${url.changefreq}</changefreq>` : ''}
    ${url.priority ? `<priority>${url.priority}</priority>` : ''}
  </url>`).join('\n')}
</urlset>`;
}

Canonical URLs

function SEOHead({ page }: { page: Page }) {
  const canonicalUrl = `https://example.com${page.canonicalPath || page.path}`;
  
  return (
    <Helmet>
      <link rel="canonical" href={canonicalUrl} />
      
      {/* Handle pagination */}
      {page.pagination?.prevPage && (
        <link rel="prev" href={`${canonicalUrl}?page=${page.pagination.prevPage}`} />
      )}
      {page.pagination?.nextPage && (
        <link rel="next" href={`${canonicalUrl}?page=${page.pagination.nextPage}`} />
      )}
      
      {/* Alternate language versions */}
      <link rel="alternate" hrefLang="en" href={`https://example.com/en${page.path}`} />
      <link rel="alternate" hrefLang="ar" href={`https://example.com/ar${page.path}`} />
      <link rel="alternate" hrefLang="x-default" href={`https://example.com${page.path}`} />
    </Helmet>
  );
}

Structured Data (Schema.org)

Organization and Website Schema

function OrganizationSchema() {
  const schema = {
    '@context': 'https://schema.org',
    '@type': 'Organization',
    name: 'Company Name',
    url: 'https://example.com',
    logo: 'https://example.com/logo.png',
    sameAs: [
      'https://twitter.com/company',
      'https://linkedin.com/company/company',
      'https://github.com/company',
    ],
    contactPoint: {
      '@type': 'ContactPoint',
      telephone: '+1-234-567-8900',
      contactType: 'customer service',
      availableLanguage: ['English', 'Arabic'],
    },
  };

  return (
    <script
      type="application/ld+json"
      dangerouslySetInnerHTML={{ __html: JSON.stringify(schema) }}
    />
  );
}

Article Schema

function ArticleSchema({ article }: { article: Article }) {
  const schema = {
    '@context': 'https://schema.org',
    '@type': 'TechArticle',
    headline: article.title,
    description: article.description,
    image: article.featuredImage,
    datePublished: article.publishedAt,
    dateModified: article.updatedAt || article.publishedAt,
    author: {
      '@type': 'Person',
      name: article.author.name,
      url: article.author.url,
    },
    publisher: {
      '@type': 'Organization',
      name: 'Company Name',
      logo: {
        '@type': 'ImageObject',
        url: 'https://example.com/logo.png',
      },
    },
    mainEntityOfPage: {
      '@type': 'WebPage',
      '@id': `https://example.com/blog/${article.slug}`,
    },
    wordCount: article.wordCount,
    keywords: article.tags.join(', '),
  };

  return (
    <script
      type="application/ld+json"
      dangerouslySetInnerHTML={{ __html: JSON.stringify(schema) }}
    />
  );
}

Breadcrumb Schema

function BreadcrumbSchema({ items }: { items: BreadcrumbItem[] }) {
  const schema = {
    '@context': 'https://schema.org',
    '@type': 'BreadcrumbList',
    itemListElement: items.map((item, index) => ({
      '@type': 'ListItem',
      position: index + 1,
      name: item.name,
      item: item.url,
    })),
  };

  return (
    <script
      type="application/ld+json"
      dangerouslySetInnerHTML={{ __html: JSON.stringify(schema) }}
    />
  );
}

URL Architecture

Clean URL Structure

// Good URL patterns
const goodUrls = [
  '/blog/technical-seo-guide',           // Descriptive, lowercase, hyphens
  '/services/web-development',           // Hierarchical structure
  '/projects/healthcare-clinic-system',     // Meaningful slugs
];

// Bad URL patterns
const badUrls = [
  '/blog?id=123',                        // Query parameters for content
  '/Blog/Technical_SEO_Guide',           // Mixed case, underscores
  '/p/12345',                            // Non-descriptive
  '/services/web-development.html',      // Unnecessary extensions
];

// URL generation utility
function generateSlug(title: string): string {
  return title
    .toLowerCase()
    .replace(/[^a-z0-9\s-]/g, '')        // Remove special chars
    .replace(/\s+/g, '-')                // Spaces to hyphens
    .replace(/-+/g, '-')                 // Multiple hyphens to single
    .replace(/^-|-$/g, '');              // Trim hyphens
}

Redirect Management

// Redirect configuration
const redirects: Redirect[] = [
  // 301 for permanent moves
  { from: '/old-page', to: '/new-page', status: 301 },
  
  // Maintain old blog URLs
  { from: '/blog/:year/:month/:slug', to: '/blog/:slug', status: 301 },
  
  // Handle trailing slashes consistently
  { from: '/:path+/', to: '/:path+', status: 301 },
];

// Express middleware
app.use((req, res, next) => {
  const redirect = redirects.find(r => matchPath(req.path, r.from));
  
  if (redirect) {
    const newPath = applyRedirect(req.path, redirect);
    return res.redirect(redirect.status, newPath);
  }
  
  next();
});

Meta Tags

function SEOMeta({ page }: { page: PageSEO }) {
  return (
    <Helmet>
      {/* Primary Meta Tags */}
      <title>{page.title} | Site Name</title>
      <meta name="description" content={page.description} />
      <meta name="keywords" content={page.keywords?.join(', ')} />
      
      {/* Open Graph */}
      <meta property="og:type" content={page.type || 'website'} />
      <meta property="og:title" content={page.title} />
      <meta property="og:description" content={page.description} />
      <meta property="og:image" content={page.image} />
      <meta property="og:url" content={page.url} />
      <meta property="og:site_name" content="Site Name" />
      <meta property="og:locale" content={page.locale || 'en_US'} />
      
      {/* Twitter Card */}
      <meta name="twitter:card" content="summary_large_image" />
      <meta name="twitter:title" content={page.title} />
      <meta name="twitter:description" content={page.description} />
      <meta name="twitter:image" content={page.image} />
      
      {/* Robots */}
      <meta 
        name="robots" 
        content={page.noindex ? 'noindex, nofollow' : 'index, follow'} 
      />
    </Helmet>
  );
}

Conclusion

Technical SEO success requires:

1. Crawlability: Proper robots.txt, XML sitemaps, and internal linking
2. Indexability: Canonical URLs, proper redirects, and meta robots
3. Structured Data: Schema.org markup for rich results
4. URL Architecture: Clean, descriptive, and hierarchical URLs
5. Meta Tags: Comprehensive and accurate page metadata
6. Performance: Fast loading times (see Core Web Vitals)

Technical SEO is not a one-time task—it requires ongoing monitoring and maintenance.