Ryan Malloy
0028738e33
Make your WordPress site irresistible. Natural SEO attraction with: - robots.txt management - sitemap.xml generation - LLMs.txt support - Google integration (Analytics, Search Console, Tag Manager) - Schema.org structured data - Open Graph / Twitter Card meta tags - AMP support - Visual elements gallery - Built-in backup/restore module Includes build.sh and .distignore for WordPress-installable release ZIPs.
461 lines
19 KiB
PHP
461 lines
19 KiB
PHP
<?php
|
|
/**
|
|
* Robots.txt Module for TigerStyle Heat
|
|
*
|
|
* Provides comprehensive robots.txt management with dynamic generation,
|
|
* sitemap integration, and intelligent defaults for WordPress sites.
|
|
*/
|
|
|
|
// Prevent direct access
|
|
if (!defined('ABSPATH')) {
|
|
exit;
|
|
}
|
|
|
|
class TigerStyleSEO_RobotsTxt {
|
|
|
|
/**
|
|
* Single instance
|
|
*/
|
|
private static $instance = null;
|
|
|
|
/**
|
|
* Option name for robots.txt settings
|
|
*/
|
|
private $option_name = 'tigerstyle_heat_robots_txt';
|
|
|
|
/**
|
|
* Default robots.txt rules
|
|
*/
|
|
private $default_rules = array();
|
|
|
|
/**
|
|
* Get instance
|
|
*/
|
|
public static function instance() {
|
|
if (is_null(self::$instance)) {
|
|
self::$instance = new self();
|
|
}
|
|
return self::$instance;
|
|
}
|
|
|
|
/**
|
|
* Constructor
|
|
*/
|
|
private function __construct() {
|
|
$this->init_default_rules();
|
|
$this->init_hooks();
|
|
}
|
|
|
|
/**
|
|
* Initialize WordPress hooks
|
|
*/
|
|
private function init_hooks() {
|
|
// Handle robots.txt generation
|
|
add_action('do_robotstxt', array($this, 'generate_robotstxt'), 10, 1);
|
|
|
|
// Override WordPress default robots.txt
|
|
add_filter('robots_txt', array($this, 'filter_robots_txt'), 10, 2);
|
|
|
|
// Handle admin form submissions
|
|
add_action('admin_post_tigerstyle_save_robots_txt', array($this, 'save_settings'));
|
|
|
|
// Add admin scripts
|
|
add_action('admin_enqueue_scripts', array($this, 'enqueue_admin_scripts'));
|
|
}
|
|
|
|
/**
|
|
* Initialize default robots.txt rules
|
|
*/
|
|
private function init_default_rules() {
|
|
$this->default_rules = array(
|
|
'user_agent_all' => array(
|
|
'user_agent' => '*',
|
|
'disallow' => array(
|
|
'/wp-admin/',
|
|
'/wp-includes/',
|
|
'/wp-content/plugins/',
|
|
'/wp-content/themes/',
|
|
'/wp-json/',
|
|
'/xmlrpc.php',
|
|
'/wp-*.php',
|
|
'/readme.html',
|
|
'/license.txt',
|
|
'/?s=',
|
|
'/search/',
|
|
'/author/',
|
|
'/feed/',
|
|
'/comments/',
|
|
'/trackback/',
|
|
'/wp-login.php',
|
|
'/wp-register.php'
|
|
),
|
|
'allow' => array(
|
|
'/wp-content/uploads/',
|
|
'/wp-content/themes/*/css/',
|
|
'/wp-content/themes/*/js/',
|
|
'/wp-content/themes/*/images/'
|
|
)
|
|
),
|
|
'user_agent_google_images' => array(
|
|
'user_agent' => 'Googlebot-Image',
|
|
'allow' => array(
|
|
'/wp-content/uploads/'
|
|
)
|
|
),
|
|
'crawl_delay' => 1,
|
|
'include_sitemap' => true,
|
|
'custom_rules' => ''
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get current settings
|
|
*/
|
|
private function get_settings() {
|
|
$settings = get_option($this->option_name, array());
|
|
return wp_parse_args($settings, $this->default_rules);
|
|
}
|
|
|
|
/**
|
|
* Generate robots.txt content
|
|
*/
|
|
public function generate_robotstxt($is_public) {
|
|
if (!$is_public) {
|
|
echo "User-agent: *\nDisallow: /\n";
|
|
return;
|
|
}
|
|
|
|
$settings = $this->get_settings();
|
|
$output = array();
|
|
|
|
// Add header comment
|
|
$output[] = '# Robots.txt generated by TigerStyle Heat Plugin';
|
|
$output[] = '# ' . home_url('/robots.txt');
|
|
$output[] = '# Generated on: ' . current_time('Y-m-d H:i:s T');
|
|
$output[] = '';
|
|
|
|
// Main user agent rules
|
|
if (isset($settings['user_agent_all'])) {
|
|
$rules = $settings['user_agent_all'];
|
|
$output[] = 'User-agent: ' . $rules['user_agent'];
|
|
|
|
// Disallow rules
|
|
if (!empty($rules['disallow'])) {
|
|
foreach ($rules['disallow'] as $path) {
|
|
if (!empty(trim($path))) {
|
|
$output[] = 'Disallow: ' . trim($path);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Allow rules
|
|
if (!empty($rules['allow'])) {
|
|
foreach ($rules['allow'] as $path) {
|
|
if (!empty(trim($path))) {
|
|
$output[] = 'Allow: ' . trim($path);
|
|
}
|
|
}
|
|
}
|
|
|
|
$output[] = '';
|
|
}
|
|
|
|
// Google Images specific rules
|
|
if (isset($settings['user_agent_google_images'])) {
|
|
$rules = $settings['user_agent_google_images'];
|
|
$output[] = 'User-agent: ' . $rules['user_agent'];
|
|
|
|
if (!empty($rules['allow'])) {
|
|
foreach ($rules['allow'] as $path) {
|
|
if (!empty(trim($path))) {
|
|
$output[] = 'Allow: ' . trim($path);
|
|
}
|
|
}
|
|
}
|
|
|
|
$output[] = '';
|
|
}
|
|
|
|
// Crawl delay
|
|
if (!empty($settings['crawl_delay']) && $settings['crawl_delay'] > 0) {
|
|
$output[] = 'User-agent: *';
|
|
$output[] = 'Crawl-delay: ' . intval($settings['crawl_delay']);
|
|
$output[] = '';
|
|
}
|
|
|
|
// Custom rules
|
|
if (!empty($settings['custom_rules'])) {
|
|
$output[] = '# Custom Rules';
|
|
$output[] = trim($settings['custom_rules']);
|
|
$output[] = '';
|
|
}
|
|
|
|
// Sitemap references
|
|
if (!empty($settings['include_sitemap'])) {
|
|
$output[] = '# Sitemaps';
|
|
$output[] = 'Sitemap: ' . home_url('/sitemap.xml');
|
|
$output[] = 'Sitemap: ' . home_url('/sitemap_index.xml');
|
|
|
|
// Add WordPress default sitemaps if they exist
|
|
if (function_exists('wp_sitemaps_get_server')) {
|
|
$output[] = 'Sitemap: ' . home_url('/wp-sitemap.xml');
|
|
}
|
|
}
|
|
|
|
echo implode("\n", $output);
|
|
}
|
|
|
|
/**
|
|
* Filter WordPress default robots.txt
|
|
*/
|
|
public function filter_robots_txt($output, $public) {
|
|
// Clear default output and use our custom generation
|
|
ob_start();
|
|
$this->generate_robotstxt($public);
|
|
$custom_output = ob_get_clean();
|
|
|
|
return $custom_output;
|
|
}
|
|
|
|
/**
|
|
* Save settings from admin form
|
|
*/
|
|
public function save_settings() {
|
|
// Verify nonce
|
|
if (!wp_verify_nonce($_POST['tigerstyle_robots_nonce'], 'tigerstyle_robots_save')) {
|
|
wp_die('Security check failed');
|
|
}
|
|
|
|
// Check user permissions
|
|
if (!current_user_can('manage_options')) {
|
|
wp_die('Insufficient permissions');
|
|
}
|
|
|
|
$settings = array();
|
|
|
|
// Process main user agent rules
|
|
$settings['user_agent_all'] = array(
|
|
'user_agent' => '*',
|
|
'disallow' => array_filter(array_map('trim', explode("\n", sanitize_textarea_field($_POST['disallow_rules'])))),
|
|
'allow' => array_filter(array_map('trim', explode("\n", sanitize_textarea_field($_POST['allow_rules']))))
|
|
);
|
|
|
|
// Google Images rules
|
|
$settings['user_agent_google_images'] = array(
|
|
'user_agent' => 'Googlebot-Image',
|
|
'allow' => array_filter(array_map('trim', explode("\n", sanitize_textarea_field($_POST['google_images_allow']))))
|
|
);
|
|
|
|
// Crawl delay
|
|
$settings['crawl_delay'] = intval($_POST['crawl_delay']);
|
|
|
|
// Include sitemap
|
|
$settings['include_sitemap'] = isset($_POST['include_sitemap']) ? true : false;
|
|
|
|
// Custom rules
|
|
$settings['custom_rules'] = sanitize_textarea_field($_POST['custom_rules']);
|
|
|
|
// Save settings
|
|
update_option($this->option_name, $settings);
|
|
|
|
// Redirect back with success message
|
|
wp_redirect(add_query_arg(array(
|
|
'page' => 'tigerstyle-heat',
|
|
'message' => 'robots_txt_updated'
|
|
), admin_url('admin.php')));
|
|
exit;
|
|
}
|
|
|
|
/**
|
|
* Enqueue admin scripts
|
|
*/
|
|
public function enqueue_admin_scripts($hook) {
|
|
if ($hook !== 'toplevel_page_tigerstyle-heat') {
|
|
return;
|
|
}
|
|
|
|
wp_enqueue_script('tigerstyle-robots-admin', TIGERSTYLE_HEAT_PLUGIN_URL . 'admin/js/robots-admin.js', array('jquery'), TIGERSTYLE_HEAT_VERSION, true);
|
|
}
|
|
|
|
/**
|
|
* Render admin page
|
|
*/
|
|
public function render_admin_page() {
|
|
$settings = $this->get_settings();
|
|
$current_robots_url = home_url('/robots.txt');
|
|
|
|
?>
|
|
<div class="tigerstyle-robots-container">
|
|
<div class="seo-info-box">
|
|
<h3><?php _e('Robots.txt Configuration', 'tigerstyle-heat'); ?></h3>
|
|
<p><?php _e('Configure your site\'s robots.txt file to control how search engines crawl your website.', 'tigerstyle-heat'); ?></p>
|
|
|
|
<div class="robots-current-status">
|
|
<h4><?php _e('Current Robots.txt Status', 'tigerstyle-heat'); ?></h4>
|
|
<p>
|
|
<strong><?php _e('URL:', 'tigerstyle-heat'); ?></strong>
|
|
<a href="<?php echo esc_url($current_robots_url); ?>" target="_blank"><?php echo esc_url($current_robots_url); ?></a>
|
|
<span class="dashicons dashicons-external"></span>
|
|
</p>
|
|
<button type="button" id="test-robots-btn" class="button"><?php _e('Test Current Robots.txt', 'tigerstyle-heat'); ?></button>
|
|
<div id="robots-test-result" style="margin-top: 10px;"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<form method="post" action="<?php echo admin_url('admin-post.php'); ?>">
|
|
<input type="hidden" name="action" value="tigerstyle_save_robots_txt">
|
|
<?php wp_nonce_field('tigerstyle_robots_save', 'tigerstyle_robots_nonce'); ?>
|
|
|
|
<div class="robots-config-grid">
|
|
<div class="robots-config-section">
|
|
<h4><?php _e('Disallow Rules (one per line)', 'tigerstyle-heat'); ?></h4>
|
|
<textarea name="disallow_rules" rows="10" cols="50" class="large-text code"><?php echo esc_textarea(implode("\n", $settings['user_agent_all']['disallow'])); ?></textarea>
|
|
<p class="description"><?php _e('Paths that should be blocked from crawling. Start each path with /', 'tigerstyle-heat'); ?></p>
|
|
</div>
|
|
|
|
<div class="robots-config-section">
|
|
<h4><?php _e('Allow Rules (one per line)', 'tigerstyle-heat'); ?></h4>
|
|
<textarea name="allow_rules" rows="10" cols="50" class="large-text code"><?php echo esc_textarea(implode("\n", $settings['user_agent_all']['allow'])); ?></textarea>
|
|
<p class="description"><?php _e('Paths that should be explicitly allowed for crawling.', 'tigerstyle-heat'); ?></p>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="robots-config-grid">
|
|
<div class="robots-config-section">
|
|
<h4><?php _e('Google Images Allow Rules', 'tigerstyle-heat'); ?></h4>
|
|
<textarea name="google_images_allow" rows="5" cols="50" class="large-text code"><?php echo esc_textarea(implode("\n", $settings['user_agent_google_images']['allow'])); ?></textarea>
|
|
<p class="description"><?php _e('Paths specifically allowed for Google Images crawler.', 'tigerstyle-heat'); ?></p>
|
|
</div>
|
|
|
|
<div class="robots-config-section">
|
|
<h4><?php _e('Crawl Settings', 'tigerstyle-heat'); ?></h4>
|
|
<table class="form-table">
|
|
<tr>
|
|
<th scope="row"><?php _e('Crawl Delay (seconds)', 'tigerstyle-heat'); ?></th>
|
|
<td>
|
|
<input type="number" name="crawl_delay" value="<?php echo esc_attr($settings['crawl_delay']); ?>" min="0" max="60" step="1" class="small-text">
|
|
<p class="description"><?php _e('Delay between crawler requests. 0 = no delay.', 'tigerstyle-heat'); ?></p>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<th scope="row"><?php _e('Include Sitemap References', 'tigerstyle-heat'); ?></th>
|
|
<td>
|
|
<label>
|
|
<input type="checkbox" name="include_sitemap" value="1" <?php checked($settings['include_sitemap']); ?>>
|
|
<?php _e('Automatically include sitemap URLs in robots.txt', 'tigerstyle-heat'); ?>
|
|
</label>
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="robots-config-section">
|
|
<h4><?php _e('Custom Rules', 'tigerstyle-heat'); ?></h4>
|
|
<textarea name="custom_rules" rows="8" cols="80" class="large-text code" placeholder="# Add custom robots.txt rules here"><?php echo esc_textarea($settings['custom_rules']); ?></textarea>
|
|
<p class="description"><?php _e('Add any custom robots.txt rules. These will be added after the standard rules.', 'tigerstyle-heat'); ?></p>
|
|
</div>
|
|
|
|
<div class="robots-preview-section">
|
|
<h4><?php _e('Live Preview', 'tigerstyle-heat'); ?></h4>
|
|
<button type="button" id="preview-robots-btn" class="button"><?php _e('Generate Preview', 'tigerstyle-heat'); ?></button>
|
|
<pre id="robots-preview" class="robots-preview-content"></pre>
|
|
</div>
|
|
|
|
<?php submit_button(__('Save Robots.txt Configuration', 'tigerstyle-heat'), 'primary', 'submit'); ?>
|
|
</form>
|
|
</div>
|
|
|
|
<style>
|
|
.tigerstyle-robots-container { max-width: 1200px; }
|
|
.robots-config-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin: 20px 0; }
|
|
.robots-config-section { background: #f9f9f9; padding: 15px; border-radius: 5px; }
|
|
.robots-current-status { background: #e7f3ff; padding: 15px; border-left: 4px solid #0073aa; margin: 15px 0; }
|
|
.robots-preview-content { background: #f1f1f1; padding: 15px; border: 1px solid #ddd; max-height: 300px; overflow-y: auto; }
|
|
.robots-preview-section { margin: 20px 0; }
|
|
@media (max-width: 768px) { .robots-config-grid { grid-template-columns: 1fr; } }
|
|
</style>
|
|
|
|
<script>
|
|
jQuery(document).ready(function($) {
|
|
// Test current robots.txt
|
|
$('#test-robots-btn').click(function() {
|
|
var btn = $(this);
|
|
var result = $('#robots-test-result');
|
|
|
|
btn.prop('disabled', true).text('<?php _e('Testing...', 'tigerstyle-heat'); ?>');
|
|
|
|
$.get('<?php echo esc_js($current_robots_url); ?>')
|
|
.done(function(data) {
|
|
result.html('<div class="notice notice-success inline"><p><strong><?php _e('Success!', 'tigerstyle-heat'); ?></strong> <?php _e('Robots.txt is accessible and working.', 'tigerstyle-heat'); ?></p></div>');
|
|
})
|
|
.fail(function() {
|
|
result.html('<div class="notice notice-error inline"><p><strong><?php _e('Error!', 'tigerstyle-heat'); ?></strong> <?php _e('Could not access robots.txt file.', 'tigerstyle-heat'); ?></p></div>');
|
|
})
|
|
.always(function() {
|
|
btn.prop('disabled', false).text('<?php _e('Test Current Robots.txt', 'tigerstyle-heat'); ?>');
|
|
});
|
|
});
|
|
|
|
// Generate preview
|
|
$('#preview-robots-btn').click(function() {
|
|
var formData = $('form').serialize();
|
|
var btn = $(this);
|
|
var preview = $('#robots-preview');
|
|
|
|
btn.prop('disabled', true).text('<?php _e('Generating...', 'tigerstyle-heat'); ?>');
|
|
|
|
// Simulate robots.txt generation
|
|
var disallowRules = $('textarea[name="disallow_rules"]').val().split('\n').filter(line => line.trim());
|
|
var allowRules = $('textarea[name="allow_rules"]').val().split('\n').filter(line => line.trim());
|
|
var googleImagesAllow = $('textarea[name="google_images_allow"]').val().split('\n').filter(line => line.trim());
|
|
var crawlDelay = $('input[name="crawl_delay"]').val();
|
|
var includeSitemap = $('input[name="include_sitemap"]').is(':checked');
|
|
var customRules = $('textarea[name="custom_rules"]').val();
|
|
|
|
var previewContent = '# Robots.txt generated by TigerStyle Heat Plugin\n';
|
|
previewContent += '# <?php echo esc_js($current_robots_url); ?>\n';
|
|
previewContent += '# Generated on: ' + new Date().toLocaleString() + '\n\n';
|
|
|
|
previewContent += 'User-agent: *\n';
|
|
disallowRules.forEach(rule => {
|
|
if (rule.trim()) previewContent += 'Disallow: ' + rule.trim() + '\n';
|
|
});
|
|
allowRules.forEach(rule => {
|
|
if (rule.trim()) previewContent += 'Allow: ' + rule.trim() + '\n';
|
|
});
|
|
previewContent += '\n';
|
|
|
|
if (googleImagesAllow.length > 0) {
|
|
previewContent += 'User-agent: Googlebot-Image\n';
|
|
googleImagesAllow.forEach(rule => {
|
|
if (rule.trim()) previewContent += 'Allow: ' + rule.trim() + '\n';
|
|
});
|
|
previewContent += '\n';
|
|
}
|
|
|
|
if (crawlDelay && crawlDelay > 0) {
|
|
previewContent += 'User-agent: *\n';
|
|
previewContent += 'Crawl-delay: ' + crawlDelay + '\n\n';
|
|
}
|
|
|
|
if (customRules.trim()) {
|
|
previewContent += '# Custom Rules\n';
|
|
previewContent += customRules.trim() + '\n\n';
|
|
}
|
|
|
|
if (includeSitemap) {
|
|
previewContent += '# Sitemaps\n';
|
|
previewContent += 'Sitemap: <?php echo esc_js(home_url('/sitemap.xml')); ?>\n';
|
|
previewContent += 'Sitemap: <?php echo esc_js(home_url('/sitemap_index.xml')); ?>\n';
|
|
previewContent += 'Sitemap: <?php echo esc_js(home_url('/wp-sitemap.xml')); ?>\n';
|
|
}
|
|
|
|
preview.text(previewContent);
|
|
btn.prop('disabled', false).text('<?php _e('Generate Preview', 'tigerstyle-heat'); ?>');
|
|
});
|
|
});
|
|
</script>
|
|
<?php
|
|
}
|
|
}
|